Compare commits
79 commits
main
...
studio/api
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0174461632 | ||
|
|
fc15f635ac | ||
|
|
0c9c4f20aa | ||
|
|
c1348e6ce3 | ||
|
|
8ffcba546c | ||
|
|
bba2b2bbe9 | ||
|
|
70ca96706a | ||
|
|
f1332bd050 | ||
|
|
dc2d160c64 | ||
|
|
79ddfa2ef4 | ||
|
|
bd5089d62b | ||
|
|
c7a1c37d36 | ||
|
|
84bf2744f8 | ||
|
|
2e04765258 | ||
|
|
c1f83a8c45 | ||
|
|
8616d43e31 | ||
|
|
5e090b951b | ||
|
|
41b40bff71 | ||
|
|
c81fac2c20 | ||
|
|
f281581354 | ||
|
|
5d9d1b295d | ||
|
|
a06b2a5422 | ||
|
|
df3a4582f8 | ||
|
|
69ea4e3136 | ||
|
|
542f70bc47 | ||
|
|
03c540ec79 | ||
|
|
2c74079a33 | ||
|
|
60e740e8fb | ||
|
|
384a55de43 | ||
|
|
9df8bedda2 | ||
|
|
82f637791d | ||
|
|
d0e7b1914f | ||
|
|
d8b4193b9d | ||
|
|
7d0c6b59d2 | ||
|
|
1d4d64fa60 | ||
|
|
60203ed16c | ||
|
|
917c939bc8 | ||
|
|
7340d90163 | ||
|
|
50ed0e55d8 | ||
|
|
d8169b8c6c | ||
|
|
58abf4ca50 | ||
|
|
51cfde7b23 | ||
|
|
1796ebf330 | ||
|
|
e57a5a9ce2 | ||
|
|
748b531528 | ||
|
|
2b2705bcdc | ||
|
|
68c2d6caa8 | ||
|
|
65c34fce39 | ||
|
|
c2bcb1e6e5 | ||
|
|
fc06185c12 | ||
|
|
5a15f4d099 | ||
|
|
423532cbcb | ||
|
|
221b810d52 | ||
|
|
88bf2eacfb | ||
|
|
c8a4fa0961 | ||
|
|
c7ecbfe16d | ||
|
|
e27698b3af | ||
|
|
64f642982b | ||
|
|
f8683d7897 | ||
|
|
bf8ef7fd8c | ||
|
|
082e0dd9d1 | ||
|
|
f026c640c5 | ||
|
|
29a0f2ef08 | ||
|
|
d5f90332b4 | ||
|
|
f39fd0e408 | ||
|
|
33faff1923 | ||
|
|
6ad8cb6d47 | ||
|
|
b6cb756811 | ||
|
|
a3f202a0de | ||
|
|
78ad81babd | ||
|
|
baf11a01f7 | ||
|
|
9c722b9059 | ||
|
|
0a49dfd047 | ||
|
|
8f322aa627 | ||
|
|
04e8beec62 | ||
|
|
9dee5ac2d0 | ||
|
|
79e8c7356e | ||
|
|
b5cada1028 | ||
|
|
9ec6c8bb5a |
64 changed files with 8486 additions and 1172 deletions
|
|
@ -49,7 +49,12 @@ class ApiMonitorEntry:
|
|||
status: str
|
||||
started_at: float
|
||||
updated_at: float
|
||||
# Who this row belongs to. On a shared lifecycle row it does not restrict
|
||||
# visibility (see _visible); it names the caller the row is attributed to.
|
||||
subject: Optional[str] = None
|
||||
# True for sk-unsloth key callers, not UI sessions. The floating panel only
|
||||
# auto-opens for these, so Studio's own chat does not pop it mid-chat.
|
||||
via_api_key: bool = False
|
||||
# Monotonic anchors so duration math survives wall-clock steps (NTP).
|
||||
started_monotonic: float = 0.0
|
||||
finished_monotonic: Optional[float] = None
|
||||
|
|
@ -69,7 +74,12 @@ class ApiMonitorEntry:
|
|||
# 0-100 for a running download row; None when not applicable.
|
||||
progress: Optional[float] = None
|
||||
|
||||
def snapshot(self, *, include_details: bool = True) -> dict[str, Any]:
|
||||
def snapshot(
|
||||
self,
|
||||
*,
|
||||
include_details: bool = True,
|
||||
attributed: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
duration_ms = None
|
||||
if self.finished_monotonic is not None:
|
||||
duration_ms = max(
|
||||
|
|
@ -86,6 +96,10 @@ class ApiMonitorEntry:
|
|||
"endpoint": self.endpoint,
|
||||
"method": self.method,
|
||||
"model": self.model,
|
||||
# A lifecycle row is shared, so it reaches subjects that had nothing to
|
||||
# do with it. The overlay auto-opens on this flag, so report it only to
|
||||
# the caller the row is attributed to; everyone else still sees the row.
|
||||
"via_api_key": self.via_api_key and attributed,
|
||||
"prompt_preview": _trim(self.prompt, _PREVIEW_CHARS),
|
||||
"reply_preview": _trim(self.reply, _PREVIEW_CHARS),
|
||||
"prompt_truncated": len(self.prompt) > _PREVIEW_CHARS,
|
||||
|
|
@ -120,6 +134,9 @@ class ApiMonitor:
|
|||
enabled: bool = True,
|
||||
):
|
||||
self._entries: deque[ApiMonitorEntry] = deque()
|
||||
# Shared rows one subject cleared. Deleting them would erase another
|
||||
# caller's history; keeping them makes "Clear log" look broken on reload.
|
||||
self._hidden_shared: dict[str, set[str]] = {}
|
||||
self._max_entries = max(0, max_entries)
|
||||
self._lock = threading.Lock()
|
||||
self._enabled = enabled
|
||||
|
|
@ -133,6 +150,7 @@ class ApiMonitor:
|
|||
prompt: str,
|
||||
context_length: Optional[int] = None,
|
||||
subject: Optional[str] = None,
|
||||
via_api_key: bool = False,
|
||||
) -> str:
|
||||
if not self._enabled:
|
||||
return ""
|
||||
|
|
@ -141,12 +159,15 @@ class ApiMonitor:
|
|||
id = f"apireq_{uuid.uuid4().hex[:12]}",
|
||||
endpoint = endpoint,
|
||||
method = method,
|
||||
model = model or "default",
|
||||
# str(): a raw JSON body can carry any type, and a non-string
|
||||
# breaks the UI that renders it.
|
||||
model = str(model) if model else "default",
|
||||
prompt = _trim(prompt, _MAX_PROMPT_CHARS),
|
||||
status = "running",
|
||||
started_at = now,
|
||||
updated_at = now,
|
||||
subject = subject,
|
||||
via_api_key = via_api_key,
|
||||
started_monotonic = time.monotonic(),
|
||||
context_length = context_length,
|
||||
)
|
||||
|
|
@ -162,12 +183,18 @@ class ApiMonitor:
|
|||
model: str,
|
||||
reason: Optional[str] = None,
|
||||
running: bool = False,
|
||||
via_api_key: bool = False,
|
||||
subject: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Record a model load/unload alongside the request traffic that caused it.
|
||||
|
||||
``running=True`` opens the row for the caller to close with :meth:`finish` /
|
||||
:meth:`fail`; an unload is terminal on arrival. Rows are shared (visible to
|
||||
every subject) and share the request retention budget.
|
||||
|
||||
``subject`` names the caller whose request drove this, which is what
|
||||
``via_api_key`` is reported to; it does not narrow who sees the row. Pass it
|
||||
whenever ``via_api_key`` is set, or the attribution reaches nobody.
|
||||
"""
|
||||
if not self._enabled:
|
||||
return ""
|
||||
|
|
@ -188,6 +215,14 @@ class ApiMonitor:
|
|||
event = event,
|
||||
reason = reason,
|
||||
shared = True,
|
||||
# The overlay opens on API-key traffic only. A switch or download that
|
||||
# is refused never reaches api_monitor.start, so this row is the whole
|
||||
# trace of it, and without the attribution the monitor stayed shut on
|
||||
# exactly the failures it exists to surface.
|
||||
via_api_key = via_api_key,
|
||||
# Shared rows are read by every subject, so the attribution needs an
|
||||
# owner or the pop-open lands in browsers that did not cause it.
|
||||
subject = subject,
|
||||
)
|
||||
with self._lock:
|
||||
self._entries.appendleft(entry)
|
||||
|
|
@ -354,7 +389,10 @@ class ApiMonitor:
|
|||
) -> list[dict[str, Any]]:
|
||||
with self._lock:
|
||||
return [
|
||||
entry.snapshot(include_details = include_details)
|
||||
entry.snapshot(
|
||||
include_details = include_details,
|
||||
attributed = self._attributed(entry, subject),
|
||||
)
|
||||
for entry in self._entries
|
||||
if self._visible(entry, subject)
|
||||
]
|
||||
|
|
@ -371,7 +409,10 @@ class ApiMonitor:
|
|||
return None
|
||||
if not self._visible(entry, subject):
|
||||
return None
|
||||
return entry.snapshot(include_details = True)
|
||||
return entry.snapshot(
|
||||
include_details = True,
|
||||
attributed = self._attributed(entry, subject),
|
||||
)
|
||||
|
||||
def active_count(self, *, subject: Optional[str] = None) -> int:
|
||||
# Lifecycle rows show as "running" while loading but are not in-flight API requests.
|
||||
|
|
@ -384,13 +425,48 @@ class ApiMonitor:
|
|||
and (subject is None or entry.subject == subject)
|
||||
)
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self._entries.clear()
|
||||
def clear(self, *, subject: Optional[str] = None) -> None:
|
||||
"""Drop recorded entries. ``subject`` limits the wipe to one caller's.
|
||||
|
||||
@staticmethod
|
||||
def _visible(entry: ApiMonitorEntry, subject: Optional[str]) -> bool:
|
||||
return subject is None or entry.subject == subject or entry.shared
|
||||
Every other read on this class is subject-scoped, so an unscoped clear
|
||||
would let one user erase another's history (and zero their active count
|
||||
mid-generation). Callers that genuinely mean "everything" pass None.
|
||||
"""
|
||||
with self._lock:
|
||||
if subject is None:
|
||||
self._entries.clear()
|
||||
self._hidden_shared.clear()
|
||||
return
|
||||
# A running shared row is a load in progress, not history, so it stays.
|
||||
hidden = self._hidden_shared.setdefault(subject, set())
|
||||
for entry in self._entries:
|
||||
if entry.shared and entry.status != "running":
|
||||
hidden.add(entry.id)
|
||||
# Shared rows are hidden, never dropped, even when this subject owns
|
||||
# one: they are another caller's history too, and an owned shared row
|
||||
# is exactly what an API-key load produces.
|
||||
self._entries = deque(
|
||||
entry for entry in self._entries if entry.shared or entry.subject != subject
|
||||
)
|
||||
|
||||
def _visible(self, entry: ApiMonitorEntry, subject: Optional[str]) -> bool:
|
||||
if subject is None:
|
||||
return True
|
||||
if entry.shared:
|
||||
# Shared rows reach every subject, minus the ones this one cleared.
|
||||
# Checked before ownership so clearing hides a subject's own rows too.
|
||||
return entry.id not in self._hidden_shared.get(subject, ())
|
||||
return entry.subject == subject
|
||||
|
||||
def _attributed(self, entry: ApiMonitorEntry, subject: Optional[str]) -> bool:
|
||||
"""Whether *subject* is the caller this row's API traffic belongs to.
|
||||
|
||||
Only they should have the overlay pop open for it. An unscoped read (no
|
||||
subject: internal callers and tests) sees the row's own flag.
|
||||
"""
|
||||
if subject is None:
|
||||
return True
|
||||
return entry.subject == subject
|
||||
|
||||
def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]:
|
||||
for entry in self._entries:
|
||||
|
|
@ -409,6 +485,12 @@ class ApiMonitor:
|
|||
kept.append(entry)
|
||||
terminal_seen += 1
|
||||
self._entries = kept
|
||||
# Keep hidden sets to live rows so they stay bounded by the ring buffer.
|
||||
live = {entry.id for entry in kept}
|
||||
for subject, hidden in list(self._hidden_shared.items()):
|
||||
hidden &= live
|
||||
if not hidden:
|
||||
del self._hidden_shared[subject]
|
||||
|
||||
|
||||
api_monitor = ApiMonitor(enabled = not _api_monitor_disabled())
|
||||
|
|
|
|||
|
|
@ -47,10 +47,18 @@ _WARM_DUTY = 10.0
|
|||
|
||||
def _is_abs_path_id(value: str) -> bool:
|
||||
"""True when an id is an absolute filesystem path (the ./models and LM Studio
|
||||
scanners use the on-disk path as the id) rather than a repo id like org/name."""
|
||||
from pathlib import Path
|
||||
scanners use the on-disk path as the id) rather than a repo id like org/name.
|
||||
|
||||
Both spellings count on every host. Path() follows the running OS, so a
|
||||
Windows backend read "/home/me/x.gguf" as relative and a POSIX one read
|
||||
"C:\\models\\x.gguf" the same way, and either then reached /v1/models as a
|
||||
published id. Ids outlive the machine that wrote them: settings sync, a WSL
|
||||
session and a copied config all carry the other platform's spelling, and the
|
||||
model-override identity already folds both. Neither reading can misfire on a
|
||||
repo id, which has no leading separator, drive or UNC prefix."""
|
||||
from pathlib import PurePosixPath, PureWindowsPath
|
||||
try:
|
||||
return Path(value).is_absolute()
|
||||
return PurePosixPath(value).is_absolute() or PureWindowsPath(value).is_absolute()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
|
|
|||
|
|
@ -448,6 +448,8 @@ async def maybe_auto_download(
|
|||
*,
|
||||
hf_token: Optional[str] = None,
|
||||
require_vision: bool = False,
|
||||
subject: Optional[str] = None,
|
||||
via_api_key: bool = False,
|
||||
) -> Optional[AutoDownloadRefusal]:
|
||||
"""Start (or report on) a background fetch of *requested_model*.
|
||||
|
||||
|
|
@ -457,6 +459,10 @@ async def maybe_auto_download(
|
|||
``require_vision`` refuses a target with no mmproj companion rather than spend
|
||||
gigabytes on weights that cannot answer the request; the local capability guard
|
||||
only ever sees an already-downloaded model.
|
||||
|
||||
``subject`` and ``via_api_key`` describe the caller for the monitor row this
|
||||
opens: the same /v1 endpoints serve Studio's own chat on a session JWT, so the
|
||||
download is not API-key traffic unless the request that asked for it was.
|
||||
"""
|
||||
global _active
|
||||
|
||||
|
|
@ -529,7 +535,14 @@ async def maybe_auto_download(
|
|||
|
||||
try:
|
||||
return await _admit_and_start(
|
||||
repo_id, wanted_variant, requested_model, hf_token, provisional, require_vision
|
||||
repo_id,
|
||||
wanted_variant,
|
||||
requested_model,
|
||||
hf_token,
|
||||
provisional,
|
||||
require_vision,
|
||||
subject = subject,
|
||||
via_api_key = via_api_key,
|
||||
)
|
||||
except BaseException:
|
||||
# Not `except Exception`: a cancel mid-probe would otherwise wedge the provisional slot.
|
||||
|
|
@ -544,6 +557,9 @@ async def _admit_and_start(
|
|||
hf_token: Optional[str],
|
||||
active: _Active,
|
||||
require_vision: bool = False,
|
||||
*,
|
||||
subject: Optional[str] = None,
|
||||
via_api_key: bool = False,
|
||||
) -> Optional[AutoDownloadRefusal]:
|
||||
from hub.utils.hf_errors import hf_error_status
|
||||
|
||||
|
|
@ -690,7 +706,16 @@ async def _admit_and_start(
|
|||
),
|
||||
)
|
||||
|
||||
return await _dispatch(repo_id, variant, expected_bytes, requested_model, hf_token, active)
|
||||
return await _dispatch(
|
||||
repo_id,
|
||||
variant,
|
||||
expected_bytes,
|
||||
requested_model,
|
||||
hf_token,
|
||||
active,
|
||||
subject = subject,
|
||||
via_api_key = via_api_key,
|
||||
)
|
||||
|
||||
|
||||
def preferred_quant(labels) -> Optional[str]:
|
||||
|
|
@ -737,6 +762,9 @@ async def _dispatch(
|
|||
requested_model: str,
|
||||
hf_token: Optional[str],
|
||||
active: _Active,
|
||||
*,
|
||||
subject: Optional[str] = None,
|
||||
via_api_key: bool = False,
|
||||
) -> AutoDownloadRefusal:
|
||||
global _active
|
||||
|
||||
|
|
@ -777,7 +805,18 @@ async def _dispatch(
|
|||
return busy
|
||||
|
||||
monitor_id = api_monitor.record_lifecycle(
|
||||
event = "download", model = label, reason = "api", running = True
|
||||
# Reason "api" because only a /v1 request reaches auto-download. That is not
|
||||
# the same as API-key traffic though: Studio's own chat calls those same
|
||||
# endpoints with a session JWT, and marking its download as API traffic pops
|
||||
# the overlay mid-chat, which via_api_key exists to prevent. So take the
|
||||
# attribution from the request, and name the caller it belongs to, since the
|
||||
# row is shared with every other subject.
|
||||
event = "download",
|
||||
model = label,
|
||||
reason = "api",
|
||||
running = True,
|
||||
via_api_key = via_api_key,
|
||||
subject = subject,
|
||||
)
|
||||
with _lock:
|
||||
if _active is active:
|
||||
|
|
|
|||
|
|
@ -11,13 +11,29 @@ from pydantic import BaseModel, Field, field_validator
|
|||
MAX_CHAT_TEMPLATE_BYTES = 65_536
|
||||
|
||||
|
||||
def chat_template_byte_length(value: str) -> Optional[int]:
|
||||
"""UTF-8 length, or None if the string cannot be encoded at all.
|
||||
|
||||
JSON can carry an unpaired surrogate, as a truncated emoji paste produces.
|
||||
json decodes it fine and .encode("utf-8") then raises. Callers treat None as
|
||||
"reject": such a template can never render.
|
||||
"""
|
||||
try:
|
||||
return len(value.encode("utf-8"))
|
||||
except UnicodeEncodeError:
|
||||
return None
|
||||
|
||||
|
||||
class ValidateChatTemplateRequest(BaseModel):
|
||||
template: str = Field(default = "")
|
||||
|
||||
@field_validator("template")
|
||||
@classmethod
|
||||
def _enforce_template_size(cls, value: str) -> str:
|
||||
if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES:
|
||||
size = chat_template_byte_length(value)
|
||||
if size is None:
|
||||
raise ValueError("Chat template contains unpaired surrogate characters.")
|
||||
if size > MAX_CHAT_TEMPLATE_BYTES:
|
||||
raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.")
|
||||
return value
|
||||
|
||||
|
|
|
|||
|
|
@ -1805,8 +1805,25 @@ from core.inference.anthropic_compat import (
|
|||
AnthropicStreamEmitter,
|
||||
AnthropicPassthroughEmitter,
|
||||
)
|
||||
from auth.authentication import get_current_subject
|
||||
from auth.authentication import API_KEY_PREFIX, get_current_subject
|
||||
from state import active_generations
|
||||
|
||||
|
||||
def _request_used_api_key(request: Any) -> bool:
|
||||
"""True when this request authenticated with an sk-unsloth key.
|
||||
|
||||
Studio's own chat hits these same endpoints with a session JWT, so this is
|
||||
what separates "someone is using Unsloth as an API server" from "someone is
|
||||
using Unsloth".
|
||||
"""
|
||||
try:
|
||||
header = request.headers.get("authorization") or ""
|
||||
except Exception:
|
||||
return False
|
||||
scheme, _, token = header.partition(" ")
|
||||
return scheme.lower() == "bearer" and token.startswith(API_KEY_PREFIX)
|
||||
|
||||
|
||||
from state.tool_approvals import resolve_tool_decision
|
||||
|
||||
from core.inference.key_exchange import decrypt_api_key
|
||||
|
|
@ -3864,6 +3881,7 @@ async def _maybe_auto_download_model(
|
|||
fastapi_request: Optional[Request],
|
||||
*,
|
||||
require_vision: bool = False,
|
||||
current_subject: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Opt-in: start fetching a named GGUF this server doesn't have.
|
||||
|
||||
|
|
@ -3886,6 +3904,10 @@ async def _maybe_auto_download_model(
|
|||
requested_model,
|
||||
hf_token = _auto_download_hf_token(fastapi_request),
|
||||
require_vision = require_vision,
|
||||
subject = current_subject,
|
||||
# These /v1 endpoints also serve Studio's own chat on a session JWT,
|
||||
# so only mark the download row as API traffic when it really is.
|
||||
via_api_key = _request_used_api_key(fastapi_request),
|
||||
)
|
||||
except Exception as exc:
|
||||
# Never turn a servable request into a 500 over the download attempt.
|
||||
|
|
@ -3905,6 +3927,7 @@ async def _maybe_auto_download_model(
|
|||
if isinstance(path, str)
|
||||
else refusal.message
|
||||
)
|
||||
_record_refused_request(fastapi_request, requested_model, refusal, current_subject)
|
||||
raise HTTPException(
|
||||
status_code = refusal.status,
|
||||
detail = detail,
|
||||
|
|
@ -3912,6 +3935,35 @@ async def _maybe_auto_download_model(
|
|||
)
|
||||
|
||||
|
||||
def _record_refused_request(
|
||||
fastapi_request: Optional[Request],
|
||||
requested_model: str,
|
||||
refusal: Any,
|
||||
current_subject: Optional[str],
|
||||
) -> None:
|
||||
"""Log the refused call itself, not just the download it is waiting on.
|
||||
|
||||
The refusal replaces the request, so the handler's own ``api_monitor.start``
|
||||
never runs. Only the caller that dispatched a download gets a row from
|
||||
``record_lifecycle``; anyone refused while it runs left no trace at all, and a
|
||||
download some other caller started carries their attribution, so an API-key
|
||||
client waiting on it never opened the overlay and read as Studio's own traffic.
|
||||
"""
|
||||
state = getattr(fastapi_request, "state", None)
|
||||
if getattr(state, "skip_api_monitor", False):
|
||||
return
|
||||
path = getattr(getattr(fastapi_request, "url", None), "path", None)
|
||||
entry_id = api_monitor.start(
|
||||
endpoint = path if isinstance(path, str) else "/v1",
|
||||
method = str(getattr(fastapi_request, "method", "") or "POST"),
|
||||
model = requested_model,
|
||||
prompt = "",
|
||||
subject = current_subject,
|
||||
via_api_key = _request_used_api_key(fastapi_request),
|
||||
)
|
||||
api_monitor.fail(entry_id, refusal.message)
|
||||
|
||||
|
||||
def _loaded_satisfies(requested: str) -> bool:
|
||||
"""Whether what is serving right now actually answers to *requested*.
|
||||
|
||||
|
|
@ -4214,6 +4266,7 @@ async def _maybe_auto_switch_model(
|
|||
get_openai_auto_switch_enabled,
|
||||
get_auto_unload_idle_seconds,
|
||||
get_model_override,
|
||||
model_override_load_kwargs,
|
||||
)
|
||||
from core.inference.local_model_resolver import resolve_local_gguf
|
||||
from core.inference.llama_keepwarm import (
|
||||
|
|
@ -4254,7 +4307,10 @@ async def _maybe_auto_switch_model(
|
|||
# Not on disk. Opt-in: fetch in the background and ask the caller to retry.
|
||||
if auto_switch_on and not reload_only:
|
||||
await _maybe_auto_download_model(
|
||||
requested_model, fastapi_request, require_vision = require_vision
|
||||
requested_model,
|
||||
fastapi_request,
|
||||
require_vision = require_vision,
|
||||
current_subject = current_subject,
|
||||
)
|
||||
# Idle-unload may have freed the model; reload exactly what it freed
|
||||
# (path + quant + advertised id) so an alias/unknown name stays servable
|
||||
|
|
@ -4362,22 +4418,94 @@ async def _maybe_auto_switch_model(
|
|||
if _already_serving():
|
||||
_record_serving_alias()
|
||||
return
|
||||
# Apply this model's saved launch flags so the swap honors the config.
|
||||
override = get_model_override(override_id)
|
||||
# Apply this model's saved launch config so an API swap loads
|
||||
# it exactly as the picker would. Try variant-qualified keys
|
||||
# first (two quants of one repo can differ), then bare ids, and
|
||||
# within each pair the concrete load path before the advertised
|
||||
# id. The settings UI keys every local row (a folder, an LM
|
||||
# Studio dir, a non-active HF cache, a loose .gguf) by that path,
|
||||
# while override_id is a derived alias -- the /v1/models name a
|
||||
# hand-written overrides PUT uses, and for a loose file only its
|
||||
# filename stem. Reading the alias first let an older entry under
|
||||
# it shadow the settings the user just saved, for good. A cached
|
||||
# repo is keyed by its repo id, which is override_id, and no path
|
||||
# entry exists for it, so it still resolves on the second try.
|
||||
# A standalone .gguf resolves with variant=None; an early build
|
||||
# of this feature keyed it by the quant label derived from the
|
||||
# filename, so read "<path>:LABEL" too, after the bare path the
|
||||
# picker writes today.
|
||||
file_variant = None
|
||||
if not variant and target_id.lower().endswith(".gguf"):
|
||||
from hub.utils.gguf import extract_quant_label
|
||||
file_variant = extract_quant_label(os.path.basename(target_id))
|
||||
override = {}
|
||||
for override_key in (
|
||||
f"{target_id}:{variant}" if variant else None,
|
||||
f"{override_id}:{variant}" if variant else None,
|
||||
target_id,
|
||||
f"{target_id}:{file_variant}" if file_variant else None,
|
||||
override_id,
|
||||
):
|
||||
if not override_key:
|
||||
continue
|
||||
override = get_model_override(override_key)
|
||||
if override:
|
||||
break
|
||||
load_kwargs = {"model_path": target_id, "gguf_variant": variant}
|
||||
if override.get("llama_extra_args") is not None:
|
||||
load_kwargs["llama_extra_args"] = override["llama_extra_args"]
|
||||
if override.get("max_seq_length") is not None:
|
||||
load_kwargs["max_seq_length"] = override["max_seq_length"]
|
||||
load_kwargs.update(
|
||||
model_override_load_kwargs(
|
||||
override,
|
||||
# Set for every GGUF the resolver returns; the
|
||||
# reload-stash path carries the quant it froze.
|
||||
is_gguf = bool(variant) or target_id.lower().endswith(".gguf"),
|
||||
)
|
||||
)
|
||||
saved_gpu_ids = load_kwargs.get("gpu_ids")
|
||||
if saved_gpu_ids and not await _override_gpu_ids_still_resolve(
|
||||
saved_gpu_ids
|
||||
):
|
||||
# Stale pin (GPU removed, mask changed, another host).
|
||||
# Dropping the one dead field beats 400ing the whole load.
|
||||
load_kwargs.pop("gpu_ids", None)
|
||||
logger.warning(
|
||||
"Dropping saved gpu_ids %s for %s: not available here.",
|
||||
saved_gpu_ids,
|
||||
override_id,
|
||||
)
|
||||
# Reuse the load impl so its dedup, tensor fallback, and threading
|
||||
# apply. Call the impl directly: we already hold the lifecycle gate
|
||||
# the /load route would otherwise take, so the route would deadlock.
|
||||
await _load_model_impl(
|
||||
LoadRequest(**load_kwargs),
|
||||
fastapi_request,
|
||||
current_subject,
|
||||
current_request_counted = True,
|
||||
)
|
||||
try:
|
||||
await _load_model_impl(
|
||||
LoadRequest(**load_kwargs),
|
||||
fastapi_request,
|
||||
current_subject,
|
||||
current_request_counted = True,
|
||||
)
|
||||
except HTTPException as exc:
|
||||
# The pre-flight check cannot mirror every gpu_ids rule the
|
||||
# loader applies (a Vulkan diffusion GGUF refuses GPU
|
||||
# selection outright). Retry once without the saved pin: a
|
||||
# stale placement preference must never block a request.
|
||||
if not (
|
||||
exc.status_code == 400
|
||||
and load_kwargs.get("gpu_ids")
|
||||
and "gpu" in str(exc.detail).lower()
|
||||
):
|
||||
raise
|
||||
logger.warning(
|
||||
"Retrying %s without saved gpu_ids %s: %s",
|
||||
override_id,
|
||||
load_kwargs.get("gpu_ids"),
|
||||
exc.detail,
|
||||
)
|
||||
load_kwargs.pop("gpu_ids", None)
|
||||
await _load_model_impl(
|
||||
LoadRequest(**load_kwargs),
|
||||
fastapi_request,
|
||||
current_subject,
|
||||
current_request_counted = True,
|
||||
)
|
||||
# Advertise the repo id (not the concrete load path) as the loaded
|
||||
# model's public id and override key for /v1/models and idle stash.
|
||||
get_llama_cpp_backend()._openai_advertised_id = override_id
|
||||
|
|
@ -4636,6 +4764,43 @@ def _classify_diffusion_gguf(config: ModelConfig) -> Optional[bool]:
|
|||
return True if name_says_diffusion else None
|
||||
|
||||
|
||||
async def _override_gpu_ids_still_resolve(gpu_ids: List[int]) -> bool:
|
||||
"""Whether a per-model GPU pin is usable on this machine right now.
|
||||
|
||||
normalize_model_override cannot know the device list, so it stores whatever
|
||||
was valid where the config was written. This is the load-time reconciliation
|
||||
for the device-availability rules, which are the ones that go stale.
|
||||
|
||||
Deliberately not exhaustive: model-dependent rules (a Vulkan diffusion GGUF
|
||||
refuses gpu_ids outright) need a ModelConfig this has no reason to build.
|
||||
The caller's retry-without-the-pin covers those, and covers rules added
|
||||
later, so a check missing here costs one extra attempt, not the load.
|
||||
"""
|
||||
try:
|
||||
from utils.hardware import DeviceType, get_device
|
||||
from utils.hardware.hardware import resolve_requested_gpu_ids
|
||||
|
||||
is_vulkan = LlamaCppBackend._is_vulkan_backend()
|
||||
if get_device() == DeviceType.XPU and not is_vulkan:
|
||||
# Rejected outright on XPU.
|
||||
return False
|
||||
resolved = resolve_requested_gpu_ids(gpu_ids, is_vulkan = is_vulkan)
|
||||
if is_vulkan and resolved:
|
||||
# Vulkan ordinals are their own index space, so resolve() only rejects
|
||||
# malformed ones; presence needs the ggml probe the load runs.
|
||||
binary = LlamaCppBackend._find_llama_server_binary()
|
||||
if binary:
|
||||
probed = {
|
||||
gpu[0]
|
||||
for gpu in await asyncio.to_thread(LlamaCppBackend._get_gpu_memory, binary)
|
||||
}
|
||||
if not {int(gpu_id) for gpu_id in resolved}.issubset(probed):
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def _resolve_gguf_gpu_ids_for_request(
|
||||
config: ModelConfig, gpu_ids: Optional[List[int]]
|
||||
) -> Optional[List[int]]:
|
||||
|
|
@ -5234,6 +5399,12 @@ async def _load_model_impl(
|
|||
event = "load",
|
||||
model = _lifecycle_model_label(request.model_path, request.gguf_variant),
|
||||
running = True,
|
||||
# Auto-switch loads run before the endpoint opens its request row, so a
|
||||
# load that fails there leaves this as the only trace of API traffic.
|
||||
via_api_key = _request_used_api_key(fastapi_request),
|
||||
# The row is shared, so every subject reads it. Name the caller it belongs
|
||||
# to or the overlay pops open in browsers that did not cause the traffic.
|
||||
subject = current_subject,
|
||||
)
|
||||
|
||||
native_grant_backed = False
|
||||
|
|
@ -6796,6 +6967,10 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)):
|
|||
operating_status = "idle"
|
||||
return {
|
||||
"status": operating_status,
|
||||
# The clock every entry's started_at is on. The floating monitor dates its
|
||||
# first snapshot against this instead of the browser's clock, which need not
|
||||
# agree with ours over a tunnel or from a container.
|
||||
"server_time": time.time(),
|
||||
"active_model": active_model,
|
||||
"context_length": _monitor_context_length(),
|
||||
"active_requests": active_requests,
|
||||
|
|
@ -6803,6 +6978,22 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)):
|
|||
}
|
||||
|
||||
|
||||
@studio_router.delete("/monitor")
|
||||
async def clear_api_monitor(current_subject: str = Depends(get_current_subject)):
|
||||
"""Drop this caller's recorded API history so a debugging session starts clean.
|
||||
|
||||
Scoped to the current subject, like every read on the monitor: an unscoped
|
||||
wipe would erase another user's history and zero their active-request count
|
||||
while their generation is still streaming.
|
||||
|
||||
The caller's own in-flight requests are dropped from the log too; they keep
|
||||
streaming to their client, they just stop being reported here (a later append
|
||||
re-adds nothing, since the entry id no longer resolves).
|
||||
"""
|
||||
api_monitor.clear(subject = current_subject)
|
||||
return {"cleared": True}
|
||||
|
||||
|
||||
@studio_router.get("/monitor/{entry_id}")
|
||||
async def get_api_monitor_entry(entry_id: str, current_subject: str = Depends(get_current_subject)):
|
||||
"""Return full prompt/reply details for one OpenAI-compatible API request."""
|
||||
|
|
@ -8291,6 +8482,7 @@ async def _proxy_to_external_provider(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -8851,6 +9043,7 @@ async def openai_chat_completions(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
tts_monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model_label,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -8921,6 +9114,7 @@ async def openai_chat_completions(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model_name,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -9074,6 +9268,7 @@ async def openai_chat_completions(
|
|||
if monitor_id is None and not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model_name,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -12010,6 +12205,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
|
|||
monitor_model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default")
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = monitor_model,
|
||||
prompt = prompt_text,
|
||||
|
|
@ -12272,6 +12468,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"),
|
||||
prompt = prompt_text,
|
||||
|
|
@ -12902,6 +13099,7 @@ async def _responses_non_streaming(
|
|||
monitor_id = api_monitor.start(
|
||||
endpoint = getattr(getattr(request, "url", None), "path", "/v1/responses"),
|
||||
method = getattr(request, "method", "POST"),
|
||||
via_api_key = _request_used_api_key(request),
|
||||
model = payload.model,
|
||||
prompt = _monitor_prompt_from_messages(messages),
|
||||
context_length = _monitor_context_length(),
|
||||
|
|
@ -14112,6 +14310,7 @@ async def openai_responses(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = payload.model,
|
||||
prompt = _monitor_prompt_from_messages(messages),
|
||||
|
|
@ -14597,6 +14796,7 @@ async def anthropic_messages(
|
|||
monitor_id = api_monitor.start(
|
||||
endpoint = getattr(request_url, "path", "/v1/messages"),
|
||||
method = getattr(request, "method", "POST"),
|
||||
via_api_key = _request_used_api_key(request),
|
||||
model = model_name,
|
||||
prompt = _monitor_prompt_from_messages(openai_messages),
|
||||
context_length = monitor_context_length,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import re
|
||||
from typing import Literal, Optional
|
||||
from typing import Any, Literal, Optional
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
|
@ -34,15 +34,20 @@ from utils.helper_precache_settings import (
|
|||
helper_model_disabled_by_env,
|
||||
set_helper_precache_enabled,
|
||||
)
|
||||
from picker.schemas import MAX_CHAT_TEMPLATE_BYTES, chat_template_byte_length
|
||||
from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents
|
||||
from utils.openai_auto_switch_settings import (
|
||||
DEFAULT_AUTO_UNLOAD_KEEP_KV,
|
||||
DEFAULT_OPENAI_AUTO_DOWNLOAD_ENABLED,
|
||||
DEFAULT_OPENAI_AUTO_SWITCH_ENABLED,
|
||||
MAX_GPU_ID,
|
||||
PARALLEL_SLOTS_MAX,
|
||||
PARALLEL_SLOTS_MIN,
|
||||
get_auto_unload_idle_seconds,
|
||||
get_auto_unload_keep_kv,
|
||||
get_model_overrides,
|
||||
get_openai_auto_switch_enabled,
|
||||
resolve_model_override_key,
|
||||
get_stored_auto_unload_idle_seconds,
|
||||
get_stored_openai_auto_download_enabled,
|
||||
set_model_override,
|
||||
|
|
@ -130,12 +135,102 @@ class OpenAIAutoSwitchResponse(BaseModel):
|
|||
auto_download_model: bool = DEFAULT_OPENAI_AUTO_DOWNLOAD_ENABLED
|
||||
|
||||
|
||||
# A quant suffix, as modelOverrideKey builds it. Matched against the loader's quant
|
||||
# pattern, not a length heuristic: a POSIX path may hold a colon
|
||||
# ("/models/foo:bar.gguf") and would otherwise inherit another model's flags.
|
||||
_MAX_VARIANT_SUFFIX_LEN = 64
|
||||
|
||||
# A local model's id is its path plus an optional quant suffix, and
|
||||
# LoadRequest.model_path is unbounded. A limit under PATH_MAX would 422 the server
|
||||
# sync while the local save succeeded.
|
||||
MAX_MODEL_OVERRIDE_KEY_LEN = 4096 + 1 + _MAX_VARIANT_SUFFIX_LEN
|
||||
|
||||
# normalize_model_override keeps ids 0..MAX_GPU_ID, so a longer list cannot name a
|
||||
# device the normalizer would store; it only makes it walk more duplicates. Bound
|
||||
# it here so an oversized array is rejected at the boundary instead of costing CPU.
|
||||
MAX_GPU_IDS = MAX_GPU_ID + 1
|
||||
|
||||
|
||||
class ModelOverridePayload(BaseModel):
|
||||
model_id: str = Field(..., min_length = 1)
|
||||
llama_extra_args: list[str] = Field(default_factory = list)
|
||||
"""One model's saved launch config, applied when the API loads that model.
|
||||
|
||||
Everything past ``model_id`` is optional and omitted means "app default", so a
|
||||
payload carrying only ``model_id`` clears the entry. The bounds here mirror
|
||||
``LoadRequest`` so a bad value is rejected at the boundary instead of being
|
||||
silently dropped by the normalizer; the enum-ish fields (KV dtype, speculative
|
||||
mode) are left to it, since their valid sets follow the llama.cpp build.
|
||||
"""
|
||||
|
||||
model_id: str = Field(..., min_length = 1, max_length = MAX_MODEL_OVERRIDE_KEY_LEN)
|
||||
# None means "leave the stored value alone": the settings UI has no control for
|
||||
# launch flags and must not wipe them. An explicit [] clears them (forget).
|
||||
llama_extra_args: Optional[list[str]] = None
|
||||
# ge=1: 0 is not a valid sequence length, and the setter drops a falsy value,
|
||||
# so reject it at the boundary instead of accepting then silently discarding it.
|
||||
max_seq_length: Optional[int] = Field(default = None, ge = 1, le = 1048576)
|
||||
custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576)
|
||||
kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32)
|
||||
speculative_type: Optional[str] = Field(default = None, max_length = 32)
|
||||
spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16)
|
||||
# Parallel decode slots (llama-server --parallel), GGUF-only like the picker.
|
||||
# None follows the server-wide default set at launch.
|
||||
n_parallel: Optional[int] = Field(default = None, ge = PARALLEL_SLOTS_MIN, le = PARALLEL_SLOTS_MAX)
|
||||
tensor_parallel: bool = False
|
||||
# Validated in bytes below, not by max_length: pydantic counts characters, so a
|
||||
# multi-byte template would pass here and be dropped by the UTF-8 normalizer.
|
||||
chat_template_override: Optional[str] = None
|
||||
gpu_memory_mode: Optional[Literal["auto", "manual"]] = None
|
||||
# -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset.
|
||||
gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024)
|
||||
n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024)
|
||||
gpu_ids: Optional[list[int]] = Field(default = None, max_length = MAX_GPU_IDS)
|
||||
# Explicit intent: an all-default save carries no fields, which is shape
|
||||
# identical to "forget this model". None keeps the legacy contract.
|
||||
remove: Optional[bool] = None
|
||||
# Fill in, don't replace: the one-time localStorage backfill reads the map once
|
||||
# and then writes each model in turn, so another tab saving during that pass
|
||||
# would be overwritten by this browser's older copy. The server reads and writes
|
||||
# under one transaction, which costs no extra round trip. Field level, because an
|
||||
# install upgraded from a release that stored only llama_extra_args and
|
||||
# max_seq_length holds an entry the browser has the rest of, and skipping the
|
||||
# whole entry would strand exactly the settings this migration exists to carry.
|
||||
fill_absent_fields: bool = False
|
||||
|
||||
@field_validator("chat_template_override")
|
||||
@classmethod
|
||||
def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]:
|
||||
# Mirrors LoadRequest.normalize_blank_chat_template_override.
|
||||
if value is None:
|
||||
return None
|
||||
size = chat_template_byte_length(value)
|
||||
if size is None:
|
||||
raise ValueError("Chat template contains unpaired surrogate characters.")
|
||||
if size > MAX_CHAT_TEMPLATE_BYTES:
|
||||
raise ValueError(f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit.")
|
||||
return value
|
||||
|
||||
@field_validator(
|
||||
"max_seq_length",
|
||||
"custom_context_length",
|
||||
"spec_draft_n_max",
|
||||
"n_parallel",
|
||||
"gpu_layers",
|
||||
"n_cpu_moe",
|
||||
"gpu_ids",
|
||||
mode = "before",
|
||||
)
|
||||
@classmethod
|
||||
def _no_booleans(cls, value: Any) -> Any:
|
||||
# bool subclasses int and pydantic parses non-strictly, so `true` arrives
|
||||
# as 1 and `false` as 0: a payload could pin GPU 1 or set a one-token
|
||||
# context. _bounded_int in the normalizer rejects bools for exactly that
|
||||
# reason, but never sees one, because coercion happens here first. Reject
|
||||
# only bools, so every other lax conversion the field relies on still runs.
|
||||
if isinstance(value, bool):
|
||||
raise ValueError("Expected a number, got a boolean.")
|
||||
if isinstance(value, list) and any(isinstance(item, bool) for item in value):
|
||||
raise ValueError("Expected numbers, got a boolean.")
|
||||
return value
|
||||
|
||||
|
||||
class ModelOverridesResponse(BaseModel):
|
||||
|
|
@ -294,18 +389,130 @@ def get_openai_auto_switch_overrides(
|
|||
return ModelOverridesResponse(overrides = get_model_overrides())
|
||||
|
||||
|
||||
def _bare_model_id(model_id: str) -> Optional[str]:
|
||||
"""``repo`` for a ``repo:QUANT`` key, or None when there is no quant suffix."""
|
||||
from utils.openai_auto_switch_settings import split_quant_suffix
|
||||
|
||||
# Must look like a quant, not just a short path segment. Both a bits-per-weight
|
||||
# modifier ("IQ4_XS-3.53bpw") and a stem fallback label count.
|
||||
split = split_quant_suffix(model_id)
|
||||
return split[0] if split is not None else None
|
||||
|
||||
|
||||
def _legacy_standalone_gguf_key(model_id: str) -> Optional[str]:
|
||||
"""The stored ``<path>:LABEL`` entry for a bare standalone .gguf path, if any.
|
||||
|
||||
A loose file has no quant to choose between, so it is keyed by the bare path,
|
||||
but the label derived from its filename is never empty and that is how the
|
||||
picker keyed the same file before, so an upgraded install carries entries
|
||||
under it. The auto-switch loader reads that spelling after the bare path
|
||||
misses; resolve_model_override_key does not, since folding a POSIX path only
|
||||
touches an existing suffix. None for an id that already names a quant, for a
|
||||
repo id, and when nothing is stored under the derived key.
|
||||
"""
|
||||
import os
|
||||
|
||||
if not model_id.lower().endswith(".gguf"):
|
||||
return None
|
||||
# Already qualified, so the caller named the entry it meant. Mirrors the
|
||||
# loader, which derives a label only when the resolver gave it no variant.
|
||||
if _bare_model_id(model_id) is not None:
|
||||
return None
|
||||
from hub.utils.gguf import extract_quant_label
|
||||
|
||||
label = extract_quant_label(os.path.basename(model_id))
|
||||
if not label:
|
||||
return None
|
||||
# Through the resolver rather than a raw lookup: the browser lowercases the
|
||||
# variant, and an ambiguous fold resolves to nothing rather than guessing.
|
||||
return resolve_model_override_key(f"{model_id}:{label}")
|
||||
|
||||
|
||||
@router.put("/openai-auto-switch/overrides", response_model = ModelOverridesResponse)
|
||||
def update_openai_auto_switch_override(
|
||||
payload: ModelOverridePayload, current_subject: str = Depends(get_current_subject)
|
||||
) -> ModelOverridesResponse:
|
||||
from core.inference.llama_server_args import validate_extra_args
|
||||
from utils.openai_auto_switch_settings import get_model_override
|
||||
|
||||
try:
|
||||
extra_args = validate_extra_args(payload.llama_extra_args)
|
||||
set_model_override(
|
||||
payload.model_id,
|
||||
llama_extra_args = extra_args,
|
||||
max_seq_length = payload.max_seq_length,
|
||||
if payload.fill_absent_fields and payload.remove is True:
|
||||
# A fill that is also a delete has no meaning, and silently picking one
|
||||
# would either lose settings or resurrect them.
|
||||
raise ValueError("fill_absent_fields cannot be combined with remove.")
|
||||
# Only model_id is the documented "remove". Otherwise omitted launch flags
|
||||
# carry over from the stored entry, since the settings UI cannot express them.
|
||||
requested_extra_args = payload.llama_extra_args
|
||||
# fill_absent_fields is a write mode, not a saved field: leaving it in would
|
||||
# make every payload look non-empty and break the legacy "no fields means
|
||||
# remove".
|
||||
saved_fields = payload.model_dump(
|
||||
exclude = {"model_id", "llama_extra_args", "remove", "fill_absent_fields"},
|
||||
exclude_none = True,
|
||||
)
|
||||
if payload.remove is not None:
|
||||
is_removal = payload.remove
|
||||
else:
|
||||
is_removal = not payload.tensor_parallel and not {
|
||||
key: value for key, value in saved_fields.items() if key != "tensor_parallel"
|
||||
}
|
||||
if requested_extra_args is None and not is_removal:
|
||||
stored = get_model_override(payload.model_id)
|
||||
# A fill leaves every stored value alone, so an entry that is already
|
||||
# there keeps its flags without this echoing them back through
|
||||
# validation: one accepted when it was saved but denylisted since would
|
||||
# 400 the one-time migration, which then retries on every start.
|
||||
if not (payload.fill_absent_fields and stored):
|
||||
requested_extra_args = stored.get("llama_extra_args")
|
||||
if requested_extra_args is None:
|
||||
# First per-quant save for flags stored under the bare repo id.
|
||||
# Auto-switch prefers the qualified entry, so carry them over.
|
||||
bare_id = _bare_model_id(payload.model_id)
|
||||
if bare_id:
|
||||
requested_extra_args = get_model_override(bare_id).get("llama_extra_args")
|
||||
# Not validated on an explicit remove: nothing is stored, so a 400 would only
|
||||
# leave the override in place. A stale flag must not block forgetting.
|
||||
extra_args = [] if payload.remove is True else validate_extra_args(requested_extra_args)
|
||||
if payload.remove is True:
|
||||
# An explicit remove wins over any other field in the payload. Remove the
|
||||
# key a load resolves to, not the literal one sent: the browser normalizes
|
||||
# casing before storing, so a stale entry would survive forgetting.
|
||||
target_id = resolve_model_override_key(payload.model_id) or payload.model_id
|
||||
set_model_override(target_id, llama_extra_args = [], max_seq_length = None)
|
||||
# A standalone .gguf is keyed by its bare path now, but a load also
|
||||
# reads the filename-derived <path>:LABEL entry an upgraded install
|
||||
# still holds. Clearing only what the resolver sees leaves that one
|
||||
# applying to every later API load, with the settings gone from the
|
||||
# UI and no way left to reach them.
|
||||
legacy_id = _legacy_standalone_gguf_key(payload.model_id)
|
||||
if legacy_id and legacy_id != target_id:
|
||||
set_model_override(
|
||||
legacy_id,
|
||||
llama_extra_args = [],
|
||||
max_seq_length = None,
|
||||
)
|
||||
else:
|
||||
# Save under the key a load resolves to, as the removal branch does.
|
||||
# Saving the literal id leaves two keys for one model, which makes every
|
||||
# other casing ambiguous and silently loses the settings.
|
||||
target_id = resolve_model_override_key(payload.model_id) or payload.model_id
|
||||
set_model_override(
|
||||
target_id,
|
||||
llama_extra_args = extra_args,
|
||||
max_seq_length = payload.max_seq_length,
|
||||
custom_context_length = payload.custom_context_length,
|
||||
kv_cache_dtype = payload.kv_cache_dtype,
|
||||
speculative_type = payload.speculative_type,
|
||||
spec_draft_n_max = payload.spec_draft_n_max,
|
||||
n_parallel = payload.n_parallel,
|
||||
tensor_parallel = payload.tensor_parallel,
|
||||
chat_template_override = payload.chat_template_override,
|
||||
gpu_memory_mode = payload.gpu_memory_mode,
|
||||
gpu_layers = payload.gpu_layers,
|
||||
n_cpu_moe = payload.n_cpu_moe,
|
||||
gpu_ids = payload.gpu_ids,
|
||||
fill_absent_fields = payload.fill_absent_fields,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise log_and_http_error(
|
||||
exc,
|
||||
|
|
|
|||
|
|
@ -2959,11 +2959,25 @@ def upsert_app_settings(settings: dict[str, Any]) -> dict[str, Any]:
|
|||
|
||||
|
||||
def upsert_app_setting_map_entry(
|
||||
key: str, entry_key: str, entry_value: dict[str, Any] | None
|
||||
key: str,
|
||||
entry_key: str,
|
||||
entry_value: dict[str, Any] | None,
|
||||
*,
|
||||
fill_absent_fields: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Set (or delete, when entry_value is falsy) one sub-entry of a dict-valued
|
||||
app setting, atomically under BEGIN IMMEDIATE so concurrent writers to other
|
||||
sub-entries cannot drop each other's updates."""
|
||||
sub-entries cannot drop each other's updates.
|
||||
|
||||
``fill_absent_fields`` writes only what is missing: the entry is created when
|
||||
it is not there, and otherwise gains the fields it does not already hold while
|
||||
every stored value is left exactly as it is. Nothing is ever deleted. The read
|
||||
and the write share this transaction, so a caller that read the map earlier
|
||||
cannot replace a value written since. Used by the one-time localStorage
|
||||
backfill, whose contract is that the server copy is the newer authority: an
|
||||
upgraded install can hold an entry with only the fields an older release knew,
|
||||
while this browser holds the rest, and entry-level skipping would strand them.
|
||||
"""
|
||||
conn = get_connection()
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
|
|
@ -2971,7 +2985,21 @@ def upsert_app_setting_map_entry(
|
|||
current = _json_loads(row["value_json"], {}) if row else {}
|
||||
if not isinstance(current, dict):
|
||||
current = {}
|
||||
if entry_value:
|
||||
if fill_absent_fields:
|
||||
if not entry_value:
|
||||
conn.rollback()
|
||||
return current
|
||||
stored = current.get(entry_key)
|
||||
if isinstance(stored, dict):
|
||||
# Stored values win field by field, so this only ever adds.
|
||||
merged = {**entry_value, **stored}
|
||||
if merged == stored:
|
||||
conn.rollback()
|
||||
return current
|
||||
current[entry_key] = merged
|
||||
else:
|
||||
current[entry_key] = entry_value
|
||||
elif entry_value:
|
||||
current[entry_key] = entry_value
|
||||
else:
|
||||
current.pop(entry_key, None)
|
||||
|
|
|
|||
|
|
@ -260,6 +260,60 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated():
|
|||
assert len(reply) == m._MAX_REPLY_CHARS and reply.endswith("...")
|
||||
|
||||
|
||||
def test_api_monitor_clear_is_scoped_to_one_subject():
|
||||
# Every other read is subject-scoped; an unscoped clear from the route would let
|
||||
# one caller erase another's history mid-generation.
|
||||
monitor = ApiMonitor(max_entries = 4)
|
||||
alice = monitor.start(
|
||||
endpoint = "/v1/chat/completions",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "alice prompt",
|
||||
subject = "alice",
|
||||
)
|
||||
bob = monitor.start(
|
||||
endpoint = "/v1/chat/completions",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "bob prompt",
|
||||
subject = "bob",
|
||||
)
|
||||
|
||||
monitor.clear(subject = "alice")
|
||||
assert monitor.snapshot(subject = "alice") == []
|
||||
assert [entry["id"] for entry in monitor.snapshot(subject = "bob")] == [bob]
|
||||
assert monitor.active_count(subject = "bob") == 1
|
||||
assert monitor.get(alice, subject = "alice") is None
|
||||
|
||||
# Passing no subject is the explicit "everything" path.
|
||||
monitor.clear()
|
||||
assert monitor.snapshot(subject = "bob") == []
|
||||
|
||||
|
||||
def test_api_monitor_records_whether_the_caller_used_an_api_key():
|
||||
# Studio's own chat hits these endpoints with a session JWT, and the floating
|
||||
# panel keys its auto-open off this flag, so mislabelling it pops the panel.
|
||||
monitor = ApiMonitor(max_entries = 4)
|
||||
ui = monitor.start(
|
||||
endpoint = "/api/inference/chat",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "hi",
|
||||
subject = "u",
|
||||
)
|
||||
api = monitor.start(
|
||||
endpoint = "/v1/chat/completions",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "hi",
|
||||
subject = "u",
|
||||
via_api_key = True,
|
||||
)
|
||||
by_id = {entry["id"]: entry for entry in monitor.snapshot(subject = "u")}
|
||||
assert by_id[ui]["via_api_key"] is False
|
||||
assert by_id[api]["via_api_key"] is True
|
||||
|
||||
|
||||
def test_api_monitor_disabled_is_noop():
|
||||
monitor = ApiMonitor(max_entries = 3, enabled = False)
|
||||
|
||||
|
|
@ -412,3 +466,124 @@ def test_request_rows_report_kind_request():
|
|||
monitor = ApiMonitor(max_entries = 2)
|
||||
monitor.start(endpoint = "/v1/chat/completions", method = "POST", model = "m", prompt = "hi")
|
||||
assert monitor.snapshot()[0]["kind"] == "request"
|
||||
|
||||
|
||||
def test_clear_hides_shared_lifecycle_rows_for_that_caller_only():
|
||||
"""A lifecycle row is shared, so it is visible to every caller but owned by
|
||||
none. A subject-scoped clear dropped only that subject's own rows, so the
|
||||
shared ones survived and the reload straight after "Clear log" brought them
|
||||
back: the button visibly did nothing to them. Dropping them outright is not
|
||||
an option either, since that erases another caller's history.
|
||||
"""
|
||||
monitor = ApiMonitor(max_entries = 10)
|
||||
mine = monitor.start(
|
||||
endpoint = "/v1/chat/completions",
|
||||
method = "POST",
|
||||
model = "org/A",
|
||||
prompt = "user: hi",
|
||||
subject = "alice",
|
||||
)
|
||||
monitor.finish(mine)
|
||||
shared = monitor.record_lifecycle(event = "unload", model = "org/A")
|
||||
|
||||
assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {mine, shared}
|
||||
assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared}
|
||||
|
||||
monitor.clear(subject = "alice")
|
||||
|
||||
assert monitor.snapshot(subject = "alice") == []
|
||||
# Hidden for alice, not deleted, so bob's view is untouched.
|
||||
assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {shared}
|
||||
assert monitor.get(shared, subject = "alice") is None
|
||||
assert monitor.get(shared, subject = "bob") is not None
|
||||
|
||||
|
||||
def test_clear_leaves_a_running_shared_row_visible():
|
||||
"""A load still in progress is live state, not history, so clearing the log
|
||||
must not hide the row that shows it."""
|
||||
monitor = ApiMonitor(max_entries = 10)
|
||||
running = monitor.record_lifecycle(event = "load", model = "org/A", running = True)
|
||||
monitor.clear(subject = "alice")
|
||||
assert {e["id"] for e in monitor.snapshot(subject = "alice")} == {running}
|
||||
|
||||
|
||||
def test_hidden_shared_ids_do_not_outlive_their_entries():
|
||||
"""The hidden set names rows that exist, so it stays bounded by the ring
|
||||
buffer instead of growing for the life of the process."""
|
||||
monitor = ApiMonitor(max_entries = 2)
|
||||
monitor.record_lifecycle(event = "unload", model = "org/A")
|
||||
monitor.clear(subject = "alice")
|
||||
assert monitor._hidden_shared.get("alice")
|
||||
for i in range(5):
|
||||
monitor.record_lifecycle(event = "unload", model = f"org/M{i}")
|
||||
assert not monitor._hidden_shared.get("alice")
|
||||
|
||||
|
||||
def test_an_api_triggered_lifecycle_row_carries_the_attribution():
|
||||
"""The overlay opens on API-key traffic only. An auto-switch or auto-download
|
||||
that is refused never reaches api_monitor.start, so the lifecycle row is the
|
||||
whole trace of that request; without the attribution the monitor stayed shut
|
||||
on exactly the failures it exists to surface."""
|
||||
monitor = ApiMonitor(max_entries = 5)
|
||||
|
||||
api_load = monitor.record_lifecycle(
|
||||
event = "load", model = "org/Repo-GGUF", running = True, via_api_key = True
|
||||
)
|
||||
monitor.record_lifecycle(event = "unload", model = "org/Repo-GGUF", reason = "idle")
|
||||
|
||||
rows = {e["id"]: e for e in monitor.snapshot()}
|
||||
assert rows[api_load]["via_api_key"] is True
|
||||
# A background unload is not API traffic and must not pop the overlay.
|
||||
idle = [e for e in rows.values() if e["event"] == "unload"]
|
||||
assert idle and all(e["via_api_key"] is False for e in idle)
|
||||
|
||||
# The failure path keeps it: failing the row must not drop the attribution.
|
||||
monitor.fail(api_load, error = "auto-switch refused")
|
||||
after = {e["id"]: e for e in monitor.snapshot()}
|
||||
assert after[api_load]["via_api_key"] is True
|
||||
assert after[api_load]["status"] == "error"
|
||||
|
||||
|
||||
def test_an_api_lifecycle_row_pops_the_overlay_only_for_its_own_caller():
|
||||
"""A lifecycle row is shared so it appears in every monitor list, and it also
|
||||
carries via_api_key, which is what the floating panel auto-opens on. Reported
|
||||
to everyone, the panel springs open in a browser that had nothing to do with
|
||||
the traffic. The row stays visible to all; only the attribution is scoped."""
|
||||
monitor = ApiMonitor(max_entries = 5)
|
||||
|
||||
row = monitor.record_lifecycle(
|
||||
event = "load",
|
||||
model = "org/Repo-GGUF",
|
||||
running = True,
|
||||
via_api_key = True,
|
||||
subject = "alice",
|
||||
)
|
||||
|
||||
mine = {e["id"]: e for e in monitor.snapshot(subject = "alice")}
|
||||
theirs = {e["id"]: e for e in monitor.snapshot(subject = "bob")}
|
||||
# Shared visibility is deliberate and must survive: bob still sees the load.
|
||||
assert row in mine and row in theirs
|
||||
assert mine[row]["via_api_key"] is True
|
||||
assert theirs[row]["via_api_key"] is False
|
||||
|
||||
# The details read is scoped the same way, so the panel cannot re-derive it.
|
||||
assert monitor.get(row, subject = "alice")["via_api_key"] is True
|
||||
assert monitor.get(row, subject = "bob")["via_api_key"] is False
|
||||
# An unscoped read (internal callers) still sees the row's own flag.
|
||||
assert monitor.get(row)["via_api_key"] is True
|
||||
|
||||
|
||||
def test_clearing_hides_a_shared_row_this_caller_owns_rather_than_deleting_it():
|
||||
"""An API-key load now owns its shared row. A subject-scoped clear drops that
|
||||
subject's rows, so without this the owner's Clear would delete a row every
|
||||
other caller can still see and wipe it out of their history too."""
|
||||
monitor = ApiMonitor(max_entries = 10)
|
||||
row = monitor.record_lifecycle(
|
||||
event = "unload", model = "org/Repo-GGUF", via_api_key = True, subject = "alice"
|
||||
)
|
||||
|
||||
monitor.clear(subject = "alice")
|
||||
|
||||
assert monitor.snapshot(subject = "alice") == []
|
||||
assert {e["id"] for e in monitor.snapshot(subject = "bob")} == {row}
|
||||
assert monitor.get(row, subject = "bob") is not None
|
||||
|
|
|
|||
|
|
@ -112,8 +112,7 @@ def _hub_error(error_type, status_code: int, message: str):
|
|||
|
||||
def test_the_hub_error_helper_carries_a_status_on_both_majors():
|
||||
# CI runs huggingface_hub 1.x and this box 0.x, and each takes only one of the
|
||||
# constructor shapes. A helper that silently dropped the response would make an
|
||||
# error-mapping test pass here and fail there.
|
||||
# constructor shapes.
|
||||
from hub.utils.hf_errors import hf_error_status
|
||||
|
||||
class _Legacy(Exception):
|
||||
|
|
@ -607,8 +606,7 @@ def test_a_hanging_auth_check_falls_through_to_the_download(hub, monkeypatch):
|
|||
|
||||
def test_a_companion_only_repo_is_not_held_at_busy(hub):
|
||||
# mmproj and MTP files are companions, not quants, so such a repo is non-servable
|
||||
# and falls through to the resident model. The busy probe accepted any .gguf, which
|
||||
# stranded that ordinary traffic behind an unrelated multi-hour download.
|
||||
# and falls through to the resident model.
|
||||
assert _run("unsloth/x-GGUF:UD-Q4_K_XL").code == "model_downloading"
|
||||
gb = 1024**3
|
||||
hub["info"] = _Info([_Sibling("mmproj-F16.gguf", gb), _Sibling("mtp-model.gguf", gb)])
|
||||
|
|
@ -684,13 +682,22 @@ class _Req:
|
|||
self.headers = headers or {}
|
||||
|
||||
|
||||
def _hook(model, request, enabled):
|
||||
def _hook(
|
||||
model,
|
||||
request,
|
||||
enabled,
|
||||
current_subject = None,
|
||||
):
|
||||
import utils.openai_auto_switch_settings as s
|
||||
|
||||
original = s.get_openai_auto_download_enabled
|
||||
s.get_openai_auto_download_enabled = lambda: enabled
|
||||
try:
|
||||
return asyncio.run(inference_route._maybe_auto_download_model(model, request))
|
||||
return asyncio.run(
|
||||
inference_route._maybe_auto_download_model(
|
||||
model, request, current_subject = current_subject
|
||||
)
|
||||
)
|
||||
finally:
|
||||
s.get_openai_auto_download_enabled = original
|
||||
|
||||
|
|
@ -726,13 +733,98 @@ def test_hook_uses_the_anthropic_envelope_on_messages(hub):
|
|||
|
||||
def test_hook_swallows_unexpected_failures(hub, monkeypatch):
|
||||
# A broken download path must not turn a servable request into a 500.
|
||||
async def _boom(model, hf_token = None):
|
||||
async def _boom(
|
||||
model,
|
||||
hf_token = None,
|
||||
**kwargs,
|
||||
):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
monkeypatch.setattr(auto_dl, "maybe_auto_download", _boom)
|
||||
assert _hook("unsloth/x-GGUF", _Req(), enabled = True) is None
|
||||
|
||||
|
||||
def _download_rows():
|
||||
from core.inference.api_monitor import api_monitor
|
||||
return [e for e in api_monitor.snapshot() if e["event"] == "download"]
|
||||
|
||||
|
||||
def test_a_ui_session_download_is_not_marked_as_api_traffic(hub):
|
||||
"""The monitor overlay auto-opens on via_api_key, which exists to separate
|
||||
"someone is serving other clients" from "someone is using Unsloth". Studio's
|
||||
own chat hits these same /v1 endpoints with a session JWT, so hardcoding the
|
||||
flag on the download row popped the panel open mid-chat."""
|
||||
from fastapi import HTTPException
|
||||
from core.inference.api_monitor import api_monitor
|
||||
|
||||
api_monitor.clear()
|
||||
with pytest.raises(HTTPException):
|
||||
# No Authorization header: the UI's session-JWT path.
|
||||
_hook("unsloth/x-GGUF", _Req(), enabled = True, current_subject = "unsloth")
|
||||
rows = _download_rows()
|
||||
assert rows and all(row["via_api_key"] is False for row in rows)
|
||||
|
||||
|
||||
def test_an_api_key_download_keeps_the_attribution_and_names_its_caller(hub):
|
||||
"""The row is shared, so it needs the subject as well: without one the
|
||||
attribution is reported to every logged-in browser instead of the caller."""
|
||||
from fastapi import HTTPException
|
||||
from auth.authentication import API_KEY_PREFIX
|
||||
from core.inference.api_monitor import api_monitor
|
||||
|
||||
api_monitor.clear()
|
||||
with pytest.raises(HTTPException):
|
||||
_hook(
|
||||
"unsloth/x-GGUF",
|
||||
_Req(headers = {"authorization": f"Bearer {API_KEY_PREFIX}abc123"}),
|
||||
enabled = True,
|
||||
current_subject = "unsloth",
|
||||
)
|
||||
rows = _download_rows()
|
||||
assert rows and all(row["via_api_key"] is True for row in rows)
|
||||
# Still shared: another subject sees the row, just not the attribution.
|
||||
others = [e for e in api_monitor.snapshot(subject = "someone-else") if e["event"] == "download"]
|
||||
assert len(others) == len(rows)
|
||||
assert all(row["via_api_key"] is False for row in others)
|
||||
|
||||
|
||||
def test_an_api_key_caller_waiting_on_someone_elses_download_gets_a_row(hub):
|
||||
"""A download started by Studio's own chat is attributed to the session, so an
|
||||
API-key client that asks for the same repo while it runs is refused before the
|
||||
handler's own api_monitor.start. Without a row of its own that call is invisible:
|
||||
the only row is the session's via_api_key=False download, so the overlay stays
|
||||
shut and the monitor presents API traffic as Studio's own."""
|
||||
from fastapi import HTTPException
|
||||
from auth.authentication import API_KEY_PREFIX
|
||||
from core.inference.api_monitor import api_monitor
|
||||
|
||||
api_monitor.clear()
|
||||
with pytest.raises(HTTPException):
|
||||
# Studio's chat (session JWT) starts the download and takes the slot.
|
||||
_hook("unsloth/x-GGUF", _Req(), enabled = True, current_subject = "unsloth")
|
||||
seeded = {row["id"] for row in api_monitor.snapshot(subject = "unsloth")}
|
||||
|
||||
with pytest.raises(HTTPException) as excinfo:
|
||||
# The adopted-download branch: same repo, an sk-unsloth key this time.
|
||||
_hook(
|
||||
"unsloth/x-GGUF",
|
||||
_Req(headers = {"authorization": f"Bearer {API_KEY_PREFIX}abc123"}),
|
||||
enabled = True,
|
||||
current_subject = "unsloth",
|
||||
)
|
||||
assert excinfo.value.status_code == 503
|
||||
|
||||
fresh = [e for e in api_monitor.snapshot(subject = "unsloth") if e["id"] not in seeded]
|
||||
# New (so the overlay counts it as unseen traffic) and attributed to this caller.
|
||||
assert [e for e in fresh if e["via_api_key"]], "the refused API-key call left no row"
|
||||
row = next(e for e in fresh if e["via_api_key"])
|
||||
assert row["endpoint"] == "/v1/chat/completions"
|
||||
assert row["status"] == "error"
|
||||
# Shared rows aside, another subject must not inherit the attribution.
|
||||
others = [e for e in api_monitor.snapshot(subject = "someone-else") if e["id"] == row["id"]]
|
||||
assert others == []
|
||||
|
||||
|
||||
def test_hook_prefers_the_hub_header_token(hub):
|
||||
from fastapi import HTTPException
|
||||
from hub.dependencies import HUB_HF_TOKEN_HEADER
|
||||
|
|
@ -1305,8 +1397,7 @@ def test_a_cold_scan_that_never_finishes_says_so_instead_of_guessing(monkeypatch
|
|||
|
||||
def test_a_refusal_is_never_swallowed_by_the_cannot_verify_handler(monkeypatch):
|
||||
# The checks run inside a broad `except Exception` that turns a failure to decide
|
||||
# into a fallthrough. An HTTPException there is a decision, but was logged as a
|
||||
# failure and answered by the resident model.
|
||||
# into a fallthrough.
|
||||
loaded = _Loaded("unsloth/A-GGUF", "UD-Q4_K_XL")
|
||||
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: loaded)
|
||||
monkeypatch.setattr(
|
||||
|
|
@ -1372,8 +1463,7 @@ def test_a_stale_index_is_refreshed_so_a_hub_download_becomes_visible(monkeypatc
|
|||
|
||||
|
||||
def test_an_id_v1_models_advertised_is_refused_before_the_resolver_warms(monkeypatch):
|
||||
# /v1/models can advertise an unloaded local GGUF while the resolver index is cold. A bare
|
||||
# id has no quant to refuse on, so without that evidence the resident model would answer.
|
||||
# /v1/models can advertise an unloaded local GGUF while the resolver index is cold.
|
||||
from core.inference import local_model_resolver as resolver
|
||||
|
||||
monkeypatch.setattr(resolver, "_scan", (0.0, {}))
|
||||
|
|
@ -1428,8 +1518,7 @@ def test_an_advertised_alias_for_the_resident_weights_is_still_served(monkeypatc
|
|||
|
||||
|
||||
def test_a_rejected_token_says_so_instead_of_asking_for_a_retry(hub):
|
||||
# Hugging Face 401s an expired X-Unsloth-HF-Token. Only 403/404 were handled, so it
|
||||
# fell through to a 503 telling the caller to retry something that cannot work.
|
||||
# Hugging Face 401s an expired X-Unsloth-HF-Token.
|
||||
from huggingface_hub.utils import HfHubHTTPError
|
||||
|
||||
hub["raise"] = _hub_error(HfHubHTTPError, 401, "unauthorized")
|
||||
|
|
@ -1504,8 +1593,7 @@ def test_a_quant_request_is_not_satisfied_by_transformers_weights(monkeypatch):
|
|||
|
||||
|
||||
def test_a_timed_out_download_keeps_the_slot_while_it_is_still_running(monkeypatch):
|
||||
# The watch window only bounds progress reporting. Releasing on the clock while
|
||||
# the worker is alive would admit a second multi-GB download beside it.
|
||||
# The watch window only bounds progress reporting.
|
||||
monkeypatch.setattr(auto_dl, "_MAX_WATCH_S", 0.0)
|
||||
monkeypatch.setattr(auto_dl, "_WATCH_POLL_S", 0.001)
|
||||
monkeypatch.setattr(auto_dl, "_TIMED_OUT_POLL_S", 0.001)
|
||||
|
|
@ -1597,8 +1685,7 @@ def test_a_remote_tag_that_names_no_quant_picks_the_preferred_one(hub):
|
|||
|
||||
def test_a_generic_gguf_advertises_the_label_the_worker_resolves(hub):
|
||||
# With no recognized quant token the extractors part ways: one takes the last
|
||||
# hyphenated segment, the plan and worker key the whole stem. Dispatching ours
|
||||
# made the worker exit with "No GGUF shards matching variant".
|
||||
# hyphenated segment, the plan and worker key the whole stem.
|
||||
from hub.utils.gguf import extract_quant_label as canonical
|
||||
from hub.utils.gguf_plan import build_gguf_variant_plans
|
||||
|
||||
|
|
@ -1686,8 +1773,7 @@ def test_a_non_quant_tag_does_not_tear_down_a_serving_quant(monkeypatch):
|
|||
|
||||
def test_the_trust_probe_never_falls_back_to_the_server_identity(hub, monkeypatch):
|
||||
# huggingface_hub treats None as "use the cached login", so only an explicit False
|
||||
# is anonymous. This probe passed None, so a caller-named repo was read with the
|
||||
# server's identity.
|
||||
# is anonymous.
|
||||
seen: list = []
|
||||
|
||||
def _probe(model_name, hf_token = None):
|
||||
|
|
@ -1757,8 +1843,7 @@ def test_the_retry_after_a_failure_is_told_instead_of_restarting_it(hub, monkeyp
|
|||
|
||||
|
||||
def test_a_completed_download_does_not_restage_the_scan_it_just_warmed(monkeypatch):
|
||||
# finalize_worker_exit invalidates and warms. A second invalidation here marks
|
||||
# that fresh scan stale and pushes a synchronous rescan onto the client's retry.
|
||||
# finalize_worker_exit invalidates and warms.
|
||||
import inspect
|
||||
|
||||
src = inspect.getsource(auto_dl._watch)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -152,6 +152,13 @@ def test_frontend_mirror_matches_shared_bounds():
|
|||
assert (int(low.group(1)), int(high.group(1))) == (PARALLEL_MIN, PARALLEL_MAX)
|
||||
|
||||
|
||||
def test_override_mirror_matches_shared_bounds():
|
||||
# The API auto-switch override map mirrors the bounds rather than importing them:
|
||||
# llama_server_args owns the extra-args allow-list that module stays out of.
|
||||
from utils.openai_auto_switch_settings import PARALLEL_SLOTS_MAX, PARALLEL_SLOTS_MIN
|
||||
assert (PARALLEL_SLOTS_MIN, PARALLEL_SLOTS_MAX) == (PARALLEL_MIN, PARALLEL_MAX)
|
||||
|
||||
|
||||
def test_preset_model_reuses_shared_bounds():
|
||||
# Bounds drifting from PARALLEL_MIN/MAX would 422 valid presets on every sync.
|
||||
from routes.chat_history import ChatPresetLoadConfig
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ per-request hot path; writes invalidate the cache.
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
|
@ -246,35 +247,427 @@ def set_openai_auto_switch(
|
|||
)
|
||||
|
||||
|
||||
# --- Per-model launch config -------------------------------------------------
|
||||
#
|
||||
# An override is the server-side twin of the UI's per-model config (the browser
|
||||
# localStorage map behind features/model-picker/model-config). The UI mirrors every
|
||||
# save here so an OpenAI-compatible API load gets the same launch settings the
|
||||
# picker would apply, rather than only the two legacy fields below.
|
||||
#
|
||||
# Legacy entries hold just {llama_extra_args, max_seq_length}. Every field is
|
||||
# optional and absent means "app default". A write replaces the fields it
|
||||
# expresses, so the route carries `llama_extra_args` over when the payload omits it.
|
||||
#
|
||||
# Known gap: the picker falls back to a global preference for GPU memory mode and
|
||||
# speculative decoding, and those globals live in browser localStorage. An override
|
||||
# stores only an explicit per-model choice, so an API load of a model that follows
|
||||
# the global gets the app default instead. Every other field matches the picker.
|
||||
|
||||
# Mirrors _valid_cache_types in core/inference/llama_cpp.py.
|
||||
VALID_KV_CACHE_DTYPES = frozenset(
|
||||
{"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"}
|
||||
)
|
||||
# Canonical values plus the legacy spellings LoadRequest still accepts.
|
||||
VALID_SPECULATIVE_TYPES = frozenset(
|
||||
{
|
||||
"auto",
|
||||
"mtp",
|
||||
"ngram",
|
||||
"mtp+ngram",
|
||||
"off",
|
||||
"default",
|
||||
"draft-mtp",
|
||||
"ngram-mod",
|
||||
"ngram-simple",
|
||||
}
|
||||
)
|
||||
# Only these consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI).
|
||||
MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"})
|
||||
VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"})
|
||||
|
||||
# Mirrors PARALLEL_MIN/MAX in core/inference/llama_server_args.py (the LoadRequest
|
||||
# n_parallel bounds). Mirrored rather than imported: that module owns the extra-args
|
||||
# allow-list this one must stay out of. test_parallel_slots_per_load.py pins them together.
|
||||
PARALLEL_SLOTS_MIN = 1
|
||||
PARALLEL_SLOTS_MAX = 64
|
||||
|
||||
MAX_SEQ_LENGTH_CEILING = 1048576
|
||||
MAX_CHAT_TEMPLATE_OVERRIDE_BYTES = 65_536
|
||||
# Highest device index a stored gpu_ids entry may name. Also bounds how many
|
||||
# distinct ids one entry can hold, which is what the payload limit is built from.
|
||||
MAX_GPU_ID = 1024
|
||||
|
||||
|
||||
def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
normalized = value.strip().lower()
|
||||
return normalized if normalized in allowed else None
|
||||
|
||||
|
||||
def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]:
|
||||
# bool subclasses int, so `gpu_ids: [true, false]` would pin GPUs 1 and 0.
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
# int(1.5) is 1, which would silently mangle a fractional context.
|
||||
if isinstance(value, float) and not value.is_integer():
|
||||
return None
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
# OverflowError is float("inf"), which json.loads accepts as `Infinity`.
|
||||
return None
|
||||
if parsed < minimum or parsed > maximum:
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate one per-model launch config, dropping anything unusable.
|
||||
|
||||
Silently drops rather than raising: an override is a convenience mirror of the
|
||||
UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id
|
||||
from another host) must not block persisting the rest or fail the API load that
|
||||
reads it. ``validate_extra_args`` is the caller's job -- it lives in the
|
||||
llama_server_args allow-list module, which this one must not import.
|
||||
"""
|
||||
entry: dict[str, Any] = {}
|
||||
|
||||
extra_args = payload.get("llama_extra_args")
|
||||
if isinstance(extra_args, (list, tuple)) and extra_args:
|
||||
entry["llama_extra_args"] = [str(arg) for arg in extra_args]
|
||||
|
||||
# 0 / negative means "unset"; the loader reads absence as the app default.
|
||||
for key in ("max_seq_length", "custom_context_length"):
|
||||
parsed = _bounded_int(payload.get(key), minimum = 1, maximum = MAX_SEQ_LENGTH_CEILING)
|
||||
if parsed:
|
||||
entry[key] = parsed
|
||||
|
||||
kv_cache_dtype = _clean_str(payload.get("kv_cache_dtype"), VALID_KV_CACHE_DTYPES)
|
||||
if kv_cache_dtype:
|
||||
entry["kv_cache_dtype"] = kv_cache_dtype
|
||||
|
||||
speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES)
|
||||
if speculative_type:
|
||||
entry["speculative_type"] = speculative_type
|
||||
# MTP-only; storing it otherwise shows an edit the loader ignores.
|
||||
if speculative_type in MTP_SPECULATIVE_TYPES:
|
||||
spec_draft_n_max = _bounded_int(payload.get("spec_draft_n_max"), minimum = 1, maximum = 16)
|
||||
if spec_draft_n_max:
|
||||
entry["spec_draft_n_max"] = spec_draft_n_max
|
||||
|
||||
# Blank means "follow the server-wide --parallel default", which is also what an
|
||||
# out-of-range value falls back to.
|
||||
n_parallel = _bounded_int(
|
||||
payload.get("n_parallel"), minimum = PARALLEL_SLOTS_MIN, maximum = PARALLEL_SLOTS_MAX
|
||||
)
|
||||
if n_parallel:
|
||||
entry["n_parallel"] = n_parallel
|
||||
|
||||
if _coerce_bool(payload.get("tensor_parallel")):
|
||||
entry["tensor_parallel"] = True
|
||||
|
||||
template = payload.get("chat_template_override")
|
||||
if isinstance(template, str) and template.strip():
|
||||
# A lone surrogate from JSON breaks encode(), and such a template can
|
||||
# never render, so drop it like any other bad field.
|
||||
try:
|
||||
template_bytes = len(template.encode("utf-8"))
|
||||
except UnicodeEncodeError:
|
||||
template_bytes = MAX_CHAT_TEMPLATE_OVERRIDE_BYTES + 1
|
||||
if template_bytes <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES:
|
||||
entry["chat_template_override"] = template
|
||||
|
||||
# Only "manual" is a real override: "auto" would pin the model and stop it
|
||||
# following the global GPU memory preference.
|
||||
if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual":
|
||||
entry["gpu_memory_mode"] = "manual"
|
||||
|
||||
# -1 is Auto (llama.cpp --fit), which is the default, so only >= 0 is stored.
|
||||
gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024)
|
||||
if gpu_layers is not None:
|
||||
entry["gpu_layers"] = gpu_layers
|
||||
|
||||
n_cpu_moe = _bounded_int(payload.get("n_cpu_moe"), minimum = 1, maximum = 1024)
|
||||
if n_cpu_moe:
|
||||
entry["n_cpu_moe"] = n_cpu_moe
|
||||
|
||||
gpu_ids = payload.get("gpu_ids")
|
||||
if isinstance(gpu_ids, (list, tuple)) and gpu_ids:
|
||||
# De-duplicate, preserving order: resolve_requested_gpu_ids rejects a repeat,
|
||||
# so storing [0, 0] would 400 every later API load of this model. Membership
|
||||
# is a set, not a scan of the list being built: an id only has to be in
|
||||
# 0..MAX_GPU_ID, so a long array walks that scan once per element.
|
||||
cleaned_ids: list[int] = []
|
||||
seen_ids: set[int] = set()
|
||||
for gid in gpu_ids:
|
||||
parsed = _bounded_int(gid, minimum = 0, maximum = MAX_GPU_ID)
|
||||
if parsed is not None and parsed not in seen_ids:
|
||||
seen_ids.add(parsed)
|
||||
cleaned_ids.append(parsed)
|
||||
if cleaned_ids:
|
||||
entry["gpu_ids"] = cleaned_ids
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Optional[int]:
|
||||
"""The ``max_seq_length`` an API load should send for this override.
|
||||
|
||||
Mirrors resolveFitMaxSeqLength in the UI (features/chat/presets/preset-policy.ts):
|
||||
under Manual GPU memory with Auto layers, llama.cpp's ``--fit`` owns context
|
||||
sizing, so the load sends the explicit context pin (or 0 to hand sizing over)
|
||||
rather than the stored max sequence length. Returns None to leave the field
|
||||
at the loader's default.
|
||||
"""
|
||||
manual_auto_layers = (
|
||||
is_gguf
|
||||
and override.get("gpu_memory_mode") == "manual"
|
||||
and override.get("gpu_layers") is None
|
||||
)
|
||||
if manual_auto_layers:
|
||||
return override.get("custom_context_length") or 0
|
||||
# max_seq_length wins where both are set. The UI only sends it for non-GGUF
|
||||
# models, so the two only collide in a hand-written or legacy entry.
|
||||
return override.get("max_seq_length") or override.get("custom_context_length")
|
||||
|
||||
|
||||
def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dict[str, Any]:
|
||||
"""Map a stored per-model config onto ``LoadRequest`` keyword arguments.
|
||||
|
||||
Mirrors the UI's load payload (features/chat/api/chat-adapter.ts) so an API
|
||||
auto-switch load and a picker load of the same model produce the same command
|
||||
line. GPU placement is GGUF-only there, so it is gated the same way here: a
|
||||
safetensors model loads through HF auto-placement and must not inherit a
|
||||
hidden GGUF GPU pin.
|
||||
"""
|
||||
if not override:
|
||||
return {}
|
||||
kwargs: dict[str, Any] = {}
|
||||
|
||||
max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf)
|
||||
if max_seq_length is not None:
|
||||
kwargs["max_seq_length"] = max_seq_length
|
||||
for source, target in (
|
||||
("llama_extra_args", "llama_extra_args"),
|
||||
("kv_cache_dtype", "cache_type_kv"),
|
||||
("speculative_type", "speculative_type"),
|
||||
("spec_draft_n_max", "spec_draft_n_max"),
|
||||
("tensor_parallel", "tensor_parallel"),
|
||||
("chat_template_override", "chat_template_override"),
|
||||
):
|
||||
if override.get(source) is not None:
|
||||
kwargs[target] = override[source]
|
||||
|
||||
if is_gguf:
|
||||
# Slots are a llama-server flag, and the picker sends them for GGUF only.
|
||||
if override.get("n_parallel") is not None:
|
||||
kwargs["n_parallel"] = override["n_parallel"]
|
||||
if override.get("gpu_memory_mode") is not None:
|
||||
kwargs["gpu_memory_mode"] = override["gpu_memory_mode"]
|
||||
if override.get("gpu_layers") is not None:
|
||||
kwargs["gpu_layers"] = override["gpu_layers"]
|
||||
if override.get("n_cpu_moe") is not None:
|
||||
kwargs["n_cpu_moe"] = override["n_cpu_moe"]
|
||||
if override.get("gpu_ids") is not None:
|
||||
kwargs["gpu_ids"] = override["gpu_ids"]
|
||||
return kwargs
|
||||
|
||||
|
||||
def _looks_like_filesystem_path(model_id: str) -> bool:
|
||||
"""True for an absolute path id, as the ./models and LM Studio scanners emit."""
|
||||
if model_id.startswith(("/", "\\")):
|
||||
return True
|
||||
# Windows drive letter, e.g. "C:\models\x.gguf".
|
||||
return len(model_id) >= 3 and model_id[1] == ":" and model_id[2] in ("\\", "/")
|
||||
|
||||
|
||||
# The three case-insensitive path shapes. Must stay in step with
|
||||
# features/hub/lib/model-identity.ts, which folds these before storing, or a
|
||||
# stored key becomes unreachable.
|
||||
_WINDOWS_DRIVE_PATH = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
_WSL_DRIVE_PATH = re.compile(r"^/mnt/[A-Za-z](?:/|$)")
|
||||
|
||||
|
||||
def _fold_case_insensitive_path(model_id: str) -> Optional[str]:
|
||||
"""``model_id`` folded for comparison, or None when the path is case-sensitive.
|
||||
|
||||
A Windows drive path, a UNC share and a WSL drive path all name one file
|
||||
whatever the casing, and the separator is interchangeable on Windows. A
|
||||
POSIX path is not: folding "/models/Foo.gguf" onto "/models/foo.gguf" would
|
||||
replay another model's context and GPU pin.
|
||||
"""
|
||||
slashed = model_id.replace("\\", "/")
|
||||
if _WINDOWS_DRIVE_PATH.match(model_id):
|
||||
minimum = 3
|
||||
elif slashed.startswith("//"):
|
||||
minimum = 2
|
||||
elif _WSL_DRIVE_PATH.match(slashed):
|
||||
minimum = 6
|
||||
else:
|
||||
return None
|
||||
trimmed = slashed
|
||||
while len(trimmed) > minimum and trimmed.endswith("/"):
|
||||
trimmed = trimmed[:-1]
|
||||
return trimmed.casefold()
|
||||
|
||||
|
||||
# A quant label may carry a bits-per-weight modifier ("IQ4_XS-3.53bpw") to keep two
|
||||
# files at the same base quant distinct. The two label helpers disagree on whether
|
||||
# to keep it, so readers of a stored key must accept both forms.
|
||||
_BPW_SUFFIX = re.compile(r"-[0-9]+(?:\.[0-9]+)?bpw$", re.IGNORECASE)
|
||||
_MAX_QUANT_SUFFIX_LEN = 64
|
||||
|
||||
|
||||
def split_quant_suffix(value: str) -> Optional[tuple[str, str]]:
|
||||
"""``(head, quant)`` for a ``head:QUANT`` key, or None when there is none.
|
||||
|
||||
The suffix has to be a real quant label, so an ordinary colon inside a POSIX
|
||||
filename is left alone: "/models/foo:bar.gguf" is one valid filename, and
|
||||
splitting it would graft /models/foo's launch flags onto a different model.
|
||||
"""
|
||||
from core.inference.llama_cpp import _GGUF_KNOWN_QUANT_RE
|
||||
from hub.utils.gguf import extract_quant_label
|
||||
|
||||
head, sep, tail = value.rpartition(":")
|
||||
if not sep or not head or not tail:
|
||||
return None
|
||||
if "/" in tail or "\\" in tail:
|
||||
return None
|
||||
if len(tail) <= _MAX_QUANT_SUFFIX_LEN and _GGUF_KNOWN_QUANT_RE.fullmatch(
|
||||
_BPW_SUFFIX.sub("", tail)
|
||||
):
|
||||
return head, tail
|
||||
# A .gguf with no recognizable quant token is labelled by its stem, so keys like
|
||||
# "/models/CustomModel.gguf:custommodel" exist. Storage lowercases the label
|
||||
# while the scanner keeps the filename casing, hence the case-insensitive
|
||||
# compare. Requiring exactly that label keeps an ordinary colon out:
|
||||
# "/models/foo:bar.gguf" splits to a head that is not a .gguf.
|
||||
if not head.lower().endswith(".gguf"):
|
||||
return None
|
||||
filename = head.replace("\\", "/").rsplit("/", 1)[-1]
|
||||
return (head, tail) if tail.casefold() == extract_quant_label(filename).casefold() else None
|
||||
|
||||
|
||||
def _fold_posix_path_variant(value: str) -> str:
|
||||
"""A POSIX path id with only its quant suffix folded.
|
||||
|
||||
The browser lowercases the variant but keeps the path casing, so a stored
|
||||
"/models/Foo:q4_k_m" has to be reachable from "/models/Foo:Q4_K_M" without
|
||||
also making "/models/Foo.gguf" reachable from "/models/foo.gguf".
|
||||
"""
|
||||
split = split_quant_suffix(value)
|
||||
if split is None:
|
||||
return value
|
||||
head, quant = split
|
||||
return f"{head}:{quant.casefold()}"
|
||||
|
||||
|
||||
def get_model_overrides() -> dict[str, dict]:
|
||||
"""Per-model launch overrides keyed by model id ({llama_extra_args, max_seq_length})."""
|
||||
"""Per-model launch configs keyed by model id (see normalize_model_override)."""
|
||||
raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None)
|
||||
return raw if isinstance(raw, dict) else {}
|
||||
|
||||
|
||||
def get_model_override(model_id: str) -> dict:
|
||||
"""The launch override applied when auto-switch loads ``model_id`` (or empty)."""
|
||||
override = get_model_overrides().get(model_id)
|
||||
"""The launch override applied when auto-switch loads ``model_id`` (or empty).
|
||||
|
||||
Falls back to a case-insensitive match when nothing matches exactly. Repo ids
|
||||
and quants are case-insensitive in practice ("Q4_K_M" and "q4_k_m" name one
|
||||
file), and the browser normalizes them to lowercase before storing, so an
|
||||
exact-only lookup misses entries written from that side. Exact still wins, and
|
||||
an ambiguous fallback matches nothing, so two POSIX paths differing only in
|
||||
case stay distinct.
|
||||
"""
|
||||
key = resolve_model_override_key(model_id)
|
||||
if key is None:
|
||||
return {}
|
||||
override = get_model_overrides().get(key)
|
||||
return override if isinstance(override, dict) else {}
|
||||
|
||||
|
||||
def resolve_model_override_key(model_id: str) -> Optional[str]:
|
||||
"""The stored key an override lookup for ``model_id`` would actually hit.
|
||||
|
||||
Shared by read and remove so "what a load applies" and "what forgetting this
|
||||
model clears" can never disagree.
|
||||
"""
|
||||
overrides = get_model_overrides()
|
||||
if isinstance(overrides.get(model_id), dict):
|
||||
return model_id
|
||||
if not isinstance(model_id, str):
|
||||
return None
|
||||
# A POSIX path is case-sensitive, so folding "/models/Foo.gguf" onto
|
||||
# "/models/foo.gguf" would replay another model's settings. Windows drive, UNC
|
||||
# and WSL paths are not, and the browser folds exactly those before storing, so
|
||||
# not folding them here would strand every migrated Windows entry.
|
||||
if _looks_like_filesystem_path(model_id):
|
||||
folded = _fold_case_insensitive_path(model_id)
|
||||
if folded is not None:
|
||||
|
||||
def fold(key: str) -> Optional[str]:
|
||||
return _fold_case_insensitive_path(key)
|
||||
else:
|
||||
# POSIX: the path stays case-sensitive, but the browser lowercases the
|
||||
# quant suffix, so "/models/Foo:q4_k_m" must be reachable from
|
||||
# the scanner's "/models/Foo:Q4_K_M".
|
||||
folded = _fold_posix_path_variant(model_id)
|
||||
|
||||
def fold(key: str) -> Optional[str]:
|
||||
# A path only ever folds onto another path.
|
||||
if not _looks_like_filesystem_path(key):
|
||||
return None
|
||||
return None if _fold_case_insensitive_path(key) else _fold_posix_path_variant(key)
|
||||
else:
|
||||
folded = model_id.casefold()
|
||||
|
||||
def fold(key: str) -> Optional[str]:
|
||||
# A path never folds onto a repo id: the shapes cannot collide.
|
||||
return None if _looks_like_filesystem_path(key) else key.casefold()
|
||||
|
||||
matches = [
|
||||
key
|
||||
for key, value in overrides.items()
|
||||
if isinstance(key, str) and fold(key) == folded and isinstance(value, dict)
|
||||
]
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
|
||||
|
||||
def set_model_override(
|
||||
model_id: str,
|
||||
llama_extra_args: Optional[list[str]] = None,
|
||||
max_seq_length: Optional[int] = None,
|
||||
*,
|
||||
fill_absent_fields: bool = False,
|
||||
**config: Any,
|
||||
) -> dict:
|
||||
"""Upsert one model's launch override; an override with no fields removes it."""
|
||||
"""Upsert one model's launch config; a config with no usable fields removes it.
|
||||
|
||||
The two legacy parameters stay positional for existing callers; every other
|
||||
per-model field is passed by keyword and normalized together.
|
||||
|
||||
``fill_absent_fields`` writes only what is missing: an entry already stored
|
||||
keeps every field it holds and gains only the ones it lacks. Returns the
|
||||
normalized entry either way; read the map back to see what is actually stored.
|
||||
"""
|
||||
if not model_id or not model_id.strip():
|
||||
raise ValueError("model_id is required.")
|
||||
entry: dict[str, Any] = {}
|
||||
if llama_extra_args:
|
||||
entry["llama_extra_args"] = [str(arg) for arg in llama_extra_args]
|
||||
if max_seq_length:
|
||||
entry["max_seq_length"] = max(0, int(max_seq_length))
|
||||
entry = normalize_model_override(
|
||||
{
|
||||
**config,
|
||||
"llama_extra_args": llama_extra_args,
|
||||
"max_seq_length": max_seq_length,
|
||||
}
|
||||
)
|
||||
|
||||
from storage.studio_db import upsert_app_setting_map_entry
|
||||
|
||||
# Atomic per-entry merge so two PUTs for different models can't drop each other.
|
||||
upsert_app_setting_map_entry(MODEL_OVERRIDES_SETTING_KEY, model_id.strip(), entry or None)
|
||||
upsert_app_setting_map_entry(
|
||||
MODEL_OVERRIDES_SETTING_KEY,
|
||||
model_id.strip(),
|
||||
entry or None,
|
||||
fill_absent_fields = fill_absent_fields,
|
||||
)
|
||||
_invalidate(MODEL_OVERRIDES_SETTING_KEY)
|
||||
return entry
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button";
|
|||
import { MascotImg } from "@/components/mascot-img";
|
||||
import { useT } from "@/i18n";
|
||||
import { Route as rootRoute } from "./routes/__root";
|
||||
import { Route as apiMonitorRoute } from "./routes/api";
|
||||
import { Route as dataRecipesRoute } from "./routes/data-recipes";
|
||||
import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId";
|
||||
import { Route as chatRoute } from "./routes/chat";
|
||||
|
|
@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([
|
|||
exportRoute,
|
||||
dataRecipesRoute,
|
||||
dataRecipeRoute,
|
||||
apiMonitorRoute,
|
||||
]);
|
||||
|
||||
function DefaultNotFound() {
|
||||
|
|
|
|||
|
|
@ -3,28 +3,27 @@
|
|||
|
||||
import { AppSidebar } from "@/components/app-sidebar";
|
||||
import { Navbar } from "@/components/navbar";
|
||||
import { fetchDeviceType, usePlatformStore } from "@/config/env";
|
||||
import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar";
|
||||
import {
|
||||
SettingsDialog,
|
||||
useSettingsDialogStore,
|
||||
} from "@/features/settings";
|
||||
import { fetchDeviceType, usePlatformStore } from "@/config/env";
|
||||
import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay";
|
||||
import { hasAuthToken } from "@/features/auth";
|
||||
import {
|
||||
ChatPage,
|
||||
type ChatSearch,
|
||||
clearNewChatDraft,
|
||||
StopRunningChatsDialog,
|
||||
useChatRuntimeStore,
|
||||
type ChatSearch,
|
||||
} from "@/features/chat";
|
||||
import { RemoteCodeConsentDialog } from "@/features/security";
|
||||
import { HfTokenWarningDialog } from "@/features/hf-auth";
|
||||
import { TransformersUpgradeDialog } from "@/features/transformers-upgrade";
|
||||
import { useTrainingUnloadGuard } from "@/features/training";
|
||||
import { useExportRuntimeLifecycle } from "@/features/export";
|
||||
import { hasAuthToken } from "@/features/auth";
|
||||
import { HfTokenWarningDialog } from "@/features/hf-auth";
|
||||
import { backfillModelOverrides } from "@/features/model-picker/api/migrate-model-overrides";
|
||||
import { usePersonalizationSync } from "@/features/profile";
|
||||
import { RemoteCodeConsentDialog } from "@/features/security";
|
||||
import { SettingsDialog, useSettingsDialogStore } from "@/features/settings";
|
||||
import { useTrainingUnloadGuard } from "@/features/training";
|
||||
import { TransformersUpgradeDialog } from "@/features/transformers-upgrade";
|
||||
import { useSidebarPin } from "@/hooks/use-sidebar-pin";
|
||||
import { useT, type TranslationKey } from "@/i18n";
|
||||
import { type TranslationKey, useT } from "@/i18n";
|
||||
import {
|
||||
Outlet,
|
||||
createRootRoute,
|
||||
|
|
@ -34,13 +33,7 @@ import {
|
|||
useRouterState,
|
||||
} from "@tanstack/react-router";
|
||||
import { AnimatePresence, motion } from "motion/react";
|
||||
import {
|
||||
Suspense,
|
||||
useEffect,
|
||||
useLayoutEffect,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
import { Suspense, useEffect, useLayoutEffect, useMemo, useState } from "react";
|
||||
import { AppProvider } from "../provider";
|
||||
|
||||
declare module "@tanstack/react-router" {
|
||||
|
|
@ -77,11 +70,16 @@ const CHAT_ONLY_ALLOWED = new Set([
|
|||
// Export stays reachable on chat-only hosts so the page can show its own grayed-out reason
|
||||
// instead of a silent redirect; it self-gates via export capability, so nothing runs.
|
||||
"/export",
|
||||
// Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve the
|
||||
// OpenAI-compatible API like any other host, so the monitor must be reachable
|
||||
// there or the overlay's "Expand" and the Settings > API card redirect to /chat.
|
||||
"/api-monitor",
|
||||
]);
|
||||
|
||||
function isChatOnlyAllowed(pathname: string): boolean {
|
||||
if (CHAT_ONLY_ALLOWED.has(pathname)) return true;
|
||||
if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) return true;
|
||||
if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/"))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -177,6 +175,15 @@ function RootLayout() {
|
|||
: DEFAULT_DOCUMENT_TITLE;
|
||||
}, [documentTitle]);
|
||||
|
||||
// Settings saved before the server-side override map existed live only in this
|
||||
// browser, so an API load would use app defaults. Backfill once, after auth.
|
||||
useEffect(() => {
|
||||
if (isAuthFlowRoute) {
|
||||
return;
|
||||
}
|
||||
void backfillModelOverrides();
|
||||
}, [isAuthFlowRoute]);
|
||||
|
||||
useEffect(() => {
|
||||
if (isAuthFlowRoute) {
|
||||
useSettingsDialogStore.getState().closeDialog();
|
||||
|
|
@ -225,6 +232,8 @@ function RootLayout() {
|
|||
<AppProvider>
|
||||
<PersonalizationSyncMount />
|
||||
{!isAuthFlowRoute && <SettingsDialog />}
|
||||
{/* Opens itself when API traffic arrives; hides on the full monitor page. */}
|
||||
{!isAuthFlowRoute && <ApiMonitorOverlay />}
|
||||
<HfTokenWarningDialog />
|
||||
<RemoteCodeConsentDialog />
|
||||
<TransformersUpgradeDialog />
|
||||
|
|
@ -244,7 +253,9 @@ function RootLayout() {
|
|||
className="!min-h-0 h-[calc(100dvh-var(--studio-titlebar-height,0px))] overflow-hidden"
|
||||
>
|
||||
<AppSidebar />
|
||||
<SidebarInset className={isChatRoute ? "overflow-hidden" : "overflow-y-auto"}>
|
||||
<SidebarInset
|
||||
className={isChatRoute ? "overflow-hidden" : "overflow-y-auto"}
|
||||
>
|
||||
<Navbar />
|
||||
<div
|
||||
className={`relative flex min-h-0 min-w-0 flex-1 basis-0 flex-col ${isChatRoute ? "overflow-hidden" : "overflow-visible"} ${isChatRoute ? "" : "pt-14 md:pt-[var(--studio-non-chat-content-top-inset,var(--studio-content-top-inset,0px))] md:[--studio-titlebar-height:var(--studio-non-chat-content-top-inset,var(--studio-content-top-inset,0px))]"}`}
|
||||
|
|
|
|||
21
studio/frontend/src/app/routes/api.tsx
Normal file
21
studio/frontend/src/app/routes/api.tsx
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { createRoute, lazyRouteComponent } from "@tanstack/react-router";
|
||||
import { requireAuth } from "../auth-guards";
|
||||
import { Route as rootRoute } from "./__root";
|
||||
|
||||
const ApiMonitorPage = lazyRouteComponent(
|
||||
() => import("@/features/api-monitor"),
|
||||
"ApiMonitorPage",
|
||||
);
|
||||
|
||||
export const Route = createRoute({
|
||||
getParentRoute: () => rootRoute,
|
||||
// Not "/api": the backend owns that prefix (and "/v1") and its SPA fallback 404s
|
||||
// those paths, so a deep link to /api would never reach the router.
|
||||
path: "/api-monitor",
|
||||
staticData: { title: "API" },
|
||||
beforeLoad: () => requireAuth(),
|
||||
component: ApiMonitorPage,
|
||||
});
|
||||
391
studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx
Normal file
391
studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Floating API monitor: opens itself when API traffic arrives, summarises it
|
||||
// without taking over the window, and links through to the full page.
|
||||
|
||||
import { getApiMonitor } from "@/features/chat/api/chat-api";
|
||||
import type { ApiMonitorEntry } from "@/features/chat/types/api";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
ArrowExpand01Icon,
|
||||
DragDropVerticalIcon,
|
||||
Globe02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { useNavigate, useRouterState } from "@tanstack/react-router";
|
||||
import { XIcon } from "lucide-react";
|
||||
import { AnimatePresence, motion, useDragControls } from "motion/react";
|
||||
import {
|
||||
type PointerEvent,
|
||||
type ReactElement,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { isLifecycleEntry, lifecycleLabel } from "./lifecycle";
|
||||
import {
|
||||
type ApiMonitorWatch,
|
||||
createWatch,
|
||||
observeResponse,
|
||||
rearmWatch,
|
||||
startWatching,
|
||||
} from "./new-traffic";
|
||||
import { useApiMonitorOverlayStore } from "./overlay-store";
|
||||
import { computeStats } from "./use-api-monitor";
|
||||
|
||||
// Live cadence while the panel is on screen.
|
||||
const OPEN_POLL_MS = 1500;
|
||||
// Closed, the poll only has to notice traffic started, so it backs off.
|
||||
const IDLE_POLL_MS = 5000;
|
||||
// Requests shown in the panel; the rest are one click away on the full page.
|
||||
const VISIBLE_ENTRIES = 4;
|
||||
// Quiet time before a dismissed panel re-arms.
|
||||
const REARM_QUIET_MS = 60_000;
|
||||
|
||||
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
|
||||
const V1_PREFIX_RE = /^\/v1\//;
|
||||
|
||||
function compactEndpoint(endpoint: string): string {
|
||||
return endpoint
|
||||
.replace(API_INFERENCE_PREFIX_RE, "/api")
|
||||
.replace(V1_PREFIX_RE, "/");
|
||||
}
|
||||
|
||||
function formatDuration(value?: number | null): string {
|
||||
if (value == null) {
|
||||
return "live";
|
||||
}
|
||||
if (value < 1000) {
|
||||
return `${Math.round(value)}ms`;
|
||||
}
|
||||
return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)}s`;
|
||||
}
|
||||
|
||||
function statusDotClass(status: ApiMonitorEntry["status"]): string {
|
||||
switch (status) {
|
||||
case "running":
|
||||
return "bg-blue-500 animate-pulse";
|
||||
case "error":
|
||||
return "bg-red-500";
|
||||
case "cancelled":
|
||||
return "bg-amber-500";
|
||||
default:
|
||||
return "bg-emerald-500";
|
||||
}
|
||||
}
|
||||
|
||||
function StatCell({
|
||||
label,
|
||||
value,
|
||||
tone,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
tone?: "error" | "active";
|
||||
}): ReactElement {
|
||||
return (
|
||||
<div className="flex min-w-0 flex-col items-center gap-0.5 px-1">
|
||||
<span
|
||||
className={cn(
|
||||
"truncate text-ui-15 font-semibold tabular-nums leading-none tracking-[-0.01em]",
|
||||
tone === "error" && "text-red-600 dark:text-red-400",
|
||||
tone === "active" && "text-blue-600 dark:text-blue-400",
|
||||
!tone && "text-nav-fg",
|
||||
)}
|
||||
>
|
||||
{value}
|
||||
</span>
|
||||
{/* Sentence case: metric rows read as words, not headers. */}
|
||||
<span className="truncate text-ui-11 tracking-nav text-muted-foreground">
|
||||
{label}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ApiMonitorOverlay(): ReactElement | null {
|
||||
const { isOpen, suppressed, autoOpen, open, close, setAutoOpen } =
|
||||
useApiMonitorOverlayStore();
|
||||
const navigate = useNavigate();
|
||||
const pathname = useRouterState({ select: (s) => s.location.pathname });
|
||||
const onFullPage = pathname === "/api-monitor";
|
||||
|
||||
const [data, setData] = useState<Awaited<
|
||||
ReturnType<typeof getApiMonitor>
|
||||
> | null>(null);
|
||||
|
||||
// What this session has already shown, and when it started watching.
|
||||
const watchRef = useRef<ApiMonitorWatch>(createWatch(0));
|
||||
const lastNewEntryAtRef = useRef(0);
|
||||
|
||||
// One loop for both jobs: panel contents while open, traffic watch while closed.
|
||||
// Stands down on the full page, which polls for itself.
|
||||
useEffect(() => {
|
||||
// Opted out and closed: nothing to open or show, so polling is pure load.
|
||||
if (onFullPage || (!autoOpen && !isOpen)) {
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
let timer: number | undefined;
|
||||
const intervalMs = isOpen ? OPEN_POLL_MS : IDLE_POLL_MS;
|
||||
|
||||
// Anchor the watch here, not at mount: the first snapshot can arrive much
|
||||
// later (a hidden tab skips its poll entirely, an unreachable backend fails
|
||||
// one), and everything terminal in it would otherwise read as history.
|
||||
startWatching(watchRef.current, performance.now());
|
||||
|
||||
function schedule(): void {
|
||||
timer = window.setTimeout(poll, intervalMs);
|
||||
}
|
||||
|
||||
function poll(): void {
|
||||
// A hidden tab has nobody to show the panel to.
|
||||
if (document.hidden) {
|
||||
schedule();
|
||||
return;
|
||||
}
|
||||
getApiMonitor()
|
||||
.then((next) => {
|
||||
if (!cancelled) setData(next);
|
||||
})
|
||||
.catch(() => {
|
||||
// An unreachable server is the full page's story to tell.
|
||||
if (!cancelled) setData(null);
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) schedule();
|
||||
});
|
||||
}
|
||||
|
||||
poll();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer !== undefined) window.clearTimeout(timer);
|
||||
};
|
||||
}, [isOpen, onFullPage, autoOpen]);
|
||||
|
||||
const entries = useMemo(() => data?.entries ?? [], [data]);
|
||||
const stats = useMemo(() => computeStats(entries), [entries]);
|
||||
|
||||
useEffect(() => {
|
||||
if (data == null) {
|
||||
return;
|
||||
}
|
||||
const hasNewTraffic = observeResponse(
|
||||
watchRef.current,
|
||||
data,
|
||||
performance.now(),
|
||||
);
|
||||
if (!hasNewTraffic) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const quietFor = now - lastNewEntryAtRef.current;
|
||||
lastNewEntryAtRef.current = now;
|
||||
if (!autoOpen || isOpen) {
|
||||
return;
|
||||
}
|
||||
// A dismissal holds for the burst and re-arms only once the API goes quiet.
|
||||
if (suppressed && quietFor < REARM_QUIET_MS) {
|
||||
return;
|
||||
}
|
||||
open();
|
||||
}, [data, autoOpen, suppressed, isOpen, open]);
|
||||
|
||||
// The backlog built up while the poll was stood down is not new traffic. The
|
||||
// watch re-anchors when the poll stands back up, not here, or the whole stay on
|
||||
// the full page would read as unwatched and the panel would pop with rows the
|
||||
// user has just read.
|
||||
useEffect(() => {
|
||||
if (onFullPage) {
|
||||
rearmWatch(watchRef.current);
|
||||
}
|
||||
}, [onFullPage]);
|
||||
|
||||
const [constraintsElement, setConstraintsElement] =
|
||||
useState<HTMLDivElement | null>(null);
|
||||
const constraintsRef = useMemo(
|
||||
() => ({ current: constraintsElement }),
|
||||
[constraintsElement],
|
||||
);
|
||||
const dragControls = useDragControls();
|
||||
|
||||
function startDrag(event: PointerEvent<HTMLDivElement>): void {
|
||||
event.preventDefault();
|
||||
dragControls.start(event);
|
||||
}
|
||||
|
||||
const visible = isOpen && !onFullPage;
|
||||
const serverStatus = data?.status ?? "idle";
|
||||
|
||||
return (
|
||||
<AnimatePresence>
|
||||
{visible && (
|
||||
<div
|
||||
ref={setConstraintsElement}
|
||||
className="pointer-events-none fixed inset-0 z-50"
|
||||
>
|
||||
<motion.div
|
||||
drag={true}
|
||||
dragControls={dragControls}
|
||||
dragListener={false}
|
||||
dragConstraints={constraintsRef}
|
||||
dragElastic={0}
|
||||
dragMomentum={false}
|
||||
initial={{ opacity: 0, scale: 0.94 }}
|
||||
animate={{ opacity: 1, scale: 1 }}
|
||||
exit={{ opacity: 0, scale: 0.94 }}
|
||||
/* Panel language from the sidebar's user menu and the model selector:
|
||||
menu-soft-surface, a 20px corner, and the heading font throughout. */
|
||||
className="menu-soft-surface pointer-events-auto fixed bottom-4 right-4 flex w-[400px] max-w-[calc(100vw-2rem)] cursor-default select-none resize flex-col overflow-hidden rounded-[20px] border-0 p-2.5 font-heading ring-0"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2 px-1.5 pb-2 pt-0.5">
|
||||
<div className="flex min-w-0 flex-1 items-center gap-2">
|
||||
<HugeiconsIcon
|
||||
icon={Globe02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-icon shrink-0 text-nav-fg"
|
||||
/>
|
||||
<span className="truncate text-ui-13p5 font-semibold tracking-[0.025em] text-nav-fg dark:tracking-[0.04em]">
|
||||
API monitor
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
"size-1.5 shrink-0 rounded-full",
|
||||
serverStatus === "generating"
|
||||
? "animate-pulse bg-blue-500"
|
||||
: serverStatus === "ready"
|
||||
? "bg-emerald-500"
|
||||
: "bg-muted-foreground",
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-0.5">
|
||||
<div
|
||||
onPointerDown={startDrag}
|
||||
className="flex size-7 touch-none cursor-grab items-center justify-center rounded-full text-muted-foreground transition-colors hover:bg-nav-surface-hover hover:text-foreground active:cursor-grabbing"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={DragDropVerticalIcon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={close}
|
||||
title="Close"
|
||||
aria-label="Close API monitor"
|
||||
className="flex size-7 items-center justify-center rounded-full text-muted-foreground transition-colors hover:bg-nav-surface-hover hover:text-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
>
|
||||
<XIcon className="size-3.5" strokeWidth={1.75} />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="truncate px-1.5 pb-2.5 text-ui-11p5 tracking-nav text-muted-foreground">
|
||||
{data?.active_model ?? "No model loaded"}
|
||||
</p>
|
||||
|
||||
{/* Soft tile, as the Hub and Train pages group readouts. */}
|
||||
<div className="grid grid-cols-4 rounded-[14px] bg-muted/45 py-2.5 dark:bg-background/45">
|
||||
<StatCell
|
||||
label="Live"
|
||||
value={stats.active.toLocaleString()}
|
||||
tone={stats.active > 0 ? "active" : undefined}
|
||||
/>
|
||||
<StatCell label="Requests" value={stats.total.toLocaleString()} />
|
||||
<StatCell
|
||||
label="Errors"
|
||||
value={stats.errors.toLocaleString()}
|
||||
tone={stats.errors > 0 ? "error" : undefined}
|
||||
/>
|
||||
<StatCell
|
||||
label="Avg"
|
||||
value={
|
||||
stats.avgDurationMs == null
|
||||
? "--"
|
||||
: formatDuration(stats.avgDurationMs)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Borderless rows on 12px hover pills, as in the sidebar. */}
|
||||
<div className="flex flex-col pb-1 pt-1.5">
|
||||
{entries.length === 0 ? (
|
||||
<p className="px-1.5 py-4 text-center text-ui-11p5 tracking-nav text-muted-foreground">
|
||||
No requests yet.
|
||||
</p>
|
||||
) : (
|
||||
entries.slice(0, VISIBLE_ENTRIES).map((entry) => (
|
||||
<div
|
||||
key={entry.id}
|
||||
className="flex h-9 min-w-0 items-center gap-2.5 rounded-[12px] px-3 transition-colors hover:bg-nav-surface-hover"
|
||||
>
|
||||
<span
|
||||
className={cn(
|
||||
"size-1.5 shrink-0 rounded-full",
|
||||
statusDotClass(entry.status),
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
<span className="shrink-0 truncate text-ui-12p5 font-medium tracking-nav text-nav-fg">
|
||||
{isLifecycleEntry(entry)
|
||||
? lifecycleLabel(entry)
|
||||
: compactEndpoint(entry.endpoint)}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
"min-w-0 flex-1 truncate text-ui-11p5 tracking-nav",
|
||||
entry.error
|
||||
? "text-red-600 dark:text-red-400"
|
||||
: "text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
{entry.error ? entry.error : entry.model}
|
||||
</span>
|
||||
<span className="shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{formatDuration(entry.duration_ms)}
|
||||
</span>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Through to payloads, filters and per request tokens. */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
close();
|
||||
void navigate({ to: "/api-monitor" });
|
||||
}}
|
||||
className="mt-1 flex h-[33px] w-full items-center justify-center gap-[8.5px] rounded-full bg-muted/60 text-ui-13p5 font-medium tracking-nav text-nav-fg transition-colors hover:bg-nav-surface-hover focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring dark:bg-background/50"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={ArrowExpand01Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-icon shrink-0"
|
||||
/>
|
||||
Expand to full monitor
|
||||
</button>
|
||||
|
||||
{/* Closing silences this burst; this is the permanent off. */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setAutoOpen(false);
|
||||
close();
|
||||
}}
|
||||
className="mt-1.5 w-full rounded-[12px] py-1 text-ui-11 tracking-nav text-muted-foreground transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
>
|
||||
Stop opening this automatically
|
||||
</button>
|
||||
</motion.div>
|
||||
</div>
|
||||
)}
|
||||
</AnimatePresence>
|
||||
);
|
||||
}
|
||||
914
studio/frontend/src/features/api-monitor/api-monitor-page.tsx
Normal file
914
studio/frontend/src/features/api-monitor/api-monitor-page.tsx
Normal file
|
|
@ -0,0 +1,914 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Full-page monitor for Unsloth's OpenAI-compatible API server.
|
||||
//
|
||||
// Replaces the small console buried in the API settings tab. Settings still owns
|
||||
// configuration (keys, auto-switch, examples); this page owns observability.
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { usePlatformStore } from "@/config/env";
|
||||
import { getInferenceStatus, unloadModel } from "@/features/chat/api/chat-api";
|
||||
import { resolveInferenceCheckpointId } from "@/features/chat/lib/apply-inference-status-to-store";
|
||||
import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store";
|
||||
import type { ApiMonitorEntry } from "@/features/chat/types/api";
|
||||
import { isExternalModelId } from "@/features/chat/external-providers";
|
||||
import { modelIdsMatch } from "@/features/hub/lib/model-identity";
|
||||
import { useSettingsDialogStore } from "@/features/settings";
|
||||
import { getApiBase, isTauri } from "@/lib/api-base";
|
||||
import { copyToClipboard } from "@/lib/copy-to-clipboard";
|
||||
import { Tick02Icon } from "@/lib/tick-icon";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
Copy01Icon,
|
||||
Delete02Icon,
|
||||
Globe02Icon,
|
||||
PauseIcon,
|
||||
PlayIcon,
|
||||
PowerSocket01Icon,
|
||||
RefreshIcon,
|
||||
Settings02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { type ReactElement, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { SavedModelSettingsPanel } from "./components/saved-model-settings";
|
||||
import { isLifecycleEntry, lifecycleLabel } from "./lifecycle";
|
||||
import {
|
||||
type MonitorStatusFilter,
|
||||
filterEntries,
|
||||
useApiMonitor,
|
||||
} from "./use-api-monitor";
|
||||
|
||||
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
|
||||
const V1_PREFIX_RE = /^\/v1\//;
|
||||
// Tries per revision for a detail payload. Bounded because the usual failure is
|
||||
// an entry aged out of the ring buffer, which never comes back.
|
||||
const DETAIL_FETCH_ATTEMPTS = 3;
|
||||
|
||||
const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [
|
||||
{ value: "all", label: "All requests" },
|
||||
{ value: "running", label: "In flight" },
|
||||
{ value: "completed", label: "Completed" },
|
||||
{ value: "error", label: "Errors" },
|
||||
{ value: "cancelled", label: "Cancelled" },
|
||||
];
|
||||
|
||||
function formatTime(value: number): string {
|
||||
return new Date(value * 1000).toLocaleTimeString([], {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
}
|
||||
|
||||
function formatDuration(value?: number | null): string {
|
||||
if (value == null) {
|
||||
return "running";
|
||||
}
|
||||
if (value < 1000) {
|
||||
return `${Math.round(value)} ms`;
|
||||
}
|
||||
return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`;
|
||||
}
|
||||
|
||||
function formatCount(value: number): string {
|
||||
return value.toLocaleString();
|
||||
}
|
||||
|
||||
function compactEndpoint(endpoint: string): string {
|
||||
return endpoint
|
||||
.replace(API_INFERENCE_PREFIX_RE, "/api")
|
||||
.replace(V1_PREFIX_RE, "/");
|
||||
}
|
||||
|
||||
function statusDotClass(status: ApiMonitorEntry["status"]): string {
|
||||
switch (status) {
|
||||
case "running":
|
||||
return "bg-blue-500 animate-pulse";
|
||||
case "error":
|
||||
return "bg-red-500";
|
||||
case "cancelled":
|
||||
return "bg-amber-500";
|
||||
default:
|
||||
return "bg-emerald-500";
|
||||
}
|
||||
}
|
||||
|
||||
function statusTextClass(status: ApiMonitorEntry["status"]): string {
|
||||
switch (status) {
|
||||
case "running":
|
||||
return "text-blue-600 dark:text-blue-400";
|
||||
case "error":
|
||||
return "text-red-600 dark:text-red-400";
|
||||
case "cancelled":
|
||||
return "text-amber-600 dark:text-amber-500";
|
||||
default:
|
||||
return "text-emerald-600 dark:text-emerald-500";
|
||||
}
|
||||
}
|
||||
|
||||
function StatCard({
|
||||
label,
|
||||
value,
|
||||
hint,
|
||||
tone,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
hint?: string;
|
||||
tone?: "default" | "error" | "active";
|
||||
}): ReactElement {
|
||||
return (
|
||||
<div className="flex min-w-0 flex-col gap-1 rounded-xl border border-border/60 bg-card px-4 py-3">
|
||||
<span className="truncate text-ui-11 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
{label}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
"truncate text-ui-22 font-semibold leading-tight tracking-[-0.02em]",
|
||||
tone === "error" && "text-red-600 dark:text-red-400",
|
||||
tone === "active" && "text-blue-600 dark:text-blue-400",
|
||||
)}
|
||||
>
|
||||
{value}
|
||||
</span>
|
||||
{hint ? (
|
||||
<span className="truncate text-ui-11 text-muted-foreground">
|
||||
{hint}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CopyButton({
|
||||
value,
|
||||
label,
|
||||
}: {
|
||||
value: string;
|
||||
label: string;
|
||||
}): ReactElement {
|
||||
const [copied, setCopied] = useState(false);
|
||||
// Cancel on cleanup, or navigating away mid-flash sets state after unmount.
|
||||
const timerRef = useRef<number | undefined>(undefined);
|
||||
useEffect(
|
||||
() => () => {
|
||||
if (timerRef.current !== undefined) window.clearTimeout(timerRef.current);
|
||||
},
|
||||
[],
|
||||
);
|
||||
return (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
aria-label={label}
|
||||
onClick={async () => {
|
||||
if (await copyToClipboard(value)) {
|
||||
setCopied(true);
|
||||
if (timerRef.current !== undefined) {
|
||||
window.clearTimeout(timerRef.current);
|
||||
}
|
||||
timerRef.current = window.setTimeout(() => setCopied(false), 1800);
|
||||
}
|
||||
}}
|
||||
className="h-7 shrink-0 gap-1.5 px-2 text-ui-11"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={copied ? Tick02Icon : Copy01Icon}
|
||||
strokeWidth={1.75}
|
||||
className={cn(
|
||||
"size-3.5",
|
||||
copied && "text-emerald-600 dark:text-emerald-500",
|
||||
)}
|
||||
/>
|
||||
{copied ? "Copied" : "Copy"}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
function ContextUsageBar({
|
||||
value,
|
||||
}: { value?: number | null }): ReactElement | null {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
const pct = Math.max(0, Math.min(100, Math.round(value * 100)));
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-muted">
|
||||
<div
|
||||
className={cn(
|
||||
"h-full rounded-full transition-[width]",
|
||||
// Near-full context is the usual cause of truncated replies.
|
||||
pct >= 90
|
||||
? "bg-red-500"
|
||||
: pct >= 75
|
||||
? "bg-amber-500"
|
||||
: "bg-control-accent",
|
||||
)}
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
<span className="shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{pct}%
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RequestRow({
|
||||
entry,
|
||||
selected,
|
||||
onSelect,
|
||||
}: {
|
||||
entry: ApiMonitorEntry;
|
||||
selected: boolean;
|
||||
onSelect: () => void;
|
||||
}): ReactElement {
|
||||
const preview =
|
||||
entry.error ||
|
||||
entry.reply_preview ||
|
||||
entry.prompt_preview ||
|
||||
(entry.status === "running" ? "Waiting for output…" : "No preview");
|
||||
// A load, unload or download has no prompt or reply, so it reads as a status
|
||||
// line rather than a request with a payload.
|
||||
if (isLifecycleEntry(entry)) {
|
||||
return (
|
||||
<div className="flex w-full min-w-0 flex-col gap-1 border-b border-border/50 bg-muted/25 px-4 py-3 last:border-b-0">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<span
|
||||
className={cn(
|
||||
"size-2 shrink-0 rounded-full",
|
||||
statusDotClass(entry.status),
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
<span className="truncate text-ui-13 font-medium text-foreground">
|
||||
{lifecycleLabel(entry)}
|
||||
</span>
|
||||
<span className="ml-auto shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{formatTime(entry.started_at)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="min-w-0 truncate pl-4 text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</div>
|
||||
{entry.error ? (
|
||||
<div className="min-w-0 break-words pl-4 text-ui-11 text-red-600 dark:text-red-400">
|
||||
{entry.error}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onSelect}
|
||||
aria-current={selected}
|
||||
className={cn(
|
||||
"flex w-full min-w-0 flex-col gap-1 border-b border-border/50 px-4 py-3 text-left transition-colors last:border-b-0",
|
||||
selected ? "bg-muted/70" : "hover:bg-muted/40",
|
||||
)}
|
||||
>
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<span
|
||||
className={cn(
|
||||
"size-2 shrink-0 rounded-full",
|
||||
statusDotClass(entry.status),
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
<span className="truncate text-ui-13 font-medium text-foreground">
|
||||
{compactEndpoint(entry.endpoint)}
|
||||
</span>
|
||||
<span className="ml-auto shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{formatDuration(entry.duration_ms)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center gap-2 pl-4">
|
||||
<span className="truncate text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</span>
|
||||
<span className="ml-auto shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{formatTime(entry.started_at)}
|
||||
</span>
|
||||
</div>
|
||||
<p
|
||||
className={cn(
|
||||
"line-clamp-2 pl-4 text-ui-11 leading-[1.45]",
|
||||
entry.error
|
||||
? "text-red-600 dark:text-red-400"
|
||||
: "text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
{preview}
|
||||
</p>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function PayloadBlock({
|
||||
title,
|
||||
body,
|
||||
truncated,
|
||||
loading,
|
||||
tone,
|
||||
}: {
|
||||
title: string;
|
||||
body: string;
|
||||
truncated?: boolean;
|
||||
loading?: boolean;
|
||||
tone?: "error";
|
||||
}): ReactElement {
|
||||
return (
|
||||
<section className="flex min-w-0 flex-col gap-1.5">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<h3 className="text-ui-11 font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
{title}
|
||||
</h3>
|
||||
<div className="flex items-center gap-1">
|
||||
{truncated ? (
|
||||
<span className="text-ui-10 text-muted-foreground">
|
||||
preview only
|
||||
</span>
|
||||
) : null}
|
||||
{body ? (
|
||||
<CopyButton value={body} label={`Copy ${title.toLowerCase()}`} />
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
<pre
|
||||
className={cn(
|
||||
"max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-lg bg-muted/50 p-3 text-ui-11 leading-[1.55]",
|
||||
tone === "error" && "bg-red-500/5 text-red-700 dark:text-red-400",
|
||||
)}
|
||||
>
|
||||
{loading && !body ? "Loading…" : body || "–"}
|
||||
</pre>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function RequestDetail({
|
||||
entry,
|
||||
detail,
|
||||
loading,
|
||||
}: {
|
||||
entry: ApiMonitorEntry;
|
||||
detail?: ApiMonitorEntry;
|
||||
loading: boolean;
|
||||
}): ReactElement {
|
||||
// The detail fetch is separate, so it can describe an older state of a streaming
|
||||
// entry. Prefer it only once it is as fresh as the list row, or the panel rewinds.
|
||||
const detailIsCurrent =
|
||||
detail != null &&
|
||||
detail.status === entry.status &&
|
||||
detail.updated_at >= entry.updated_at;
|
||||
const prompt = detail?.prompt ?? entry.prompt_preview;
|
||||
const reply = detailIsCurrent
|
||||
? (detail.reply ?? entry.reply_preview)
|
||||
: entry.reply_preview;
|
||||
|
||||
return (
|
||||
<div className="flex min-w-0 flex-col gap-5 p-5">
|
||||
<header className="flex min-w-0 flex-col gap-2">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<span
|
||||
className={cn(
|
||||
"size-2 shrink-0 rounded-full",
|
||||
statusDotClass(entry.status),
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
<span
|
||||
className={cn(
|
||||
"text-ui-11 font-semibold uppercase tracking-wider",
|
||||
statusTextClass(entry.status),
|
||||
)}
|
||||
>
|
||||
{entry.status}
|
||||
</span>
|
||||
<span className="ml-auto shrink-0 font-mono text-ui-10 text-muted-foreground">
|
||||
{entry.id}
|
||||
</span>
|
||||
</div>
|
||||
<h2 className="min-w-0 break-all font-mono text-ui-13 font-medium text-foreground">
|
||||
{entry.method} {entry.endpoint}
|
||||
</h2>
|
||||
<p className="min-w-0 break-all text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<dl className="grid grid-cols-2 gap-x-4 gap-y-3 rounded-xl border border-border/60 bg-card px-4 py-3 sm:grid-cols-3">
|
||||
{[
|
||||
{ label: "Started", value: formatTime(entry.started_at) },
|
||||
{ label: "Duration", value: formatDuration(entry.duration_ms) },
|
||||
{
|
||||
label: "Prompt tokens",
|
||||
value:
|
||||
entry.prompt_tokens != null
|
||||
? formatCount(entry.prompt_tokens)
|
||||
: "–",
|
||||
},
|
||||
{
|
||||
label: "Completion tokens",
|
||||
value:
|
||||
entry.completion_tokens != null
|
||||
? formatCount(entry.completion_tokens)
|
||||
: "–",
|
||||
},
|
||||
{
|
||||
label: "Total tokens",
|
||||
value:
|
||||
entry.total_tokens != null
|
||||
? formatCount(entry.total_tokens)
|
||||
: "–",
|
||||
},
|
||||
{
|
||||
label: "Context",
|
||||
value:
|
||||
entry.context_length != null
|
||||
? formatCount(entry.context_length)
|
||||
: "–",
|
||||
},
|
||||
].map((item) => (
|
||||
<div key={item.label} className="flex min-w-0 flex-col gap-0.5">
|
||||
<dt className="truncate text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
{item.label}
|
||||
</dt>
|
||||
<dd className="truncate text-ui-13 tabular-nums text-foreground">
|
||||
{item.value}
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
|
||||
{entry.context_usage != null ? (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<span className="text-ui-11 font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
Context used
|
||||
</span>
|
||||
<ContextUsageBar value={entry.context_usage} />
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{entry.error ? (
|
||||
<PayloadBlock title="Error" body={entry.error} tone="error" />
|
||||
) : null}
|
||||
|
||||
<PayloadBlock
|
||||
title="Prompt"
|
||||
body={prompt}
|
||||
truncated={entry.prompt_truncated && detail?.prompt == null}
|
||||
loading={loading}
|
||||
/>
|
||||
<PayloadBlock
|
||||
title="Reply"
|
||||
body={reply}
|
||||
truncated={entry.reply_truncated && !detailIsCurrent}
|
||||
loading={loading}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ApiMonitorPage(): ReactElement {
|
||||
const {
|
||||
data,
|
||||
entries,
|
||||
stats,
|
||||
error,
|
||||
loading,
|
||||
refreshing,
|
||||
paused,
|
||||
setPaused,
|
||||
refresh,
|
||||
clear,
|
||||
details,
|
||||
loadingDetails,
|
||||
requestDetail,
|
||||
} = useApiMonitor();
|
||||
const serverUrl = usePlatformStore((s) => s.serverUrl);
|
||||
const [unloading, setUnloading] = useState(false);
|
||||
const [unloadError, setUnloadError] = useState<string | null>(null);
|
||||
|
||||
// Manual release so VRAM is freed without waiting for the idle timer. /unload
|
||||
// matches on the internal id, which the monitor does not carry, so read status.
|
||||
const unloadActiveModel = async (): Promise<void> => {
|
||||
setUnloading(true);
|
||||
try {
|
||||
const status = await getInferenceStatus();
|
||||
const checkpoint = resolveInferenceCheckpointId(status);
|
||||
if (!checkpoint) {
|
||||
setUnloadError(null);
|
||||
return;
|
||||
}
|
||||
await unloadModel({ model_path: checkpoint });
|
||||
// Same as the chat eject flow: the store still holds the freed checkpoint.
|
||||
// Only when that IS the model just unloaded, though. Chat can have an
|
||||
// external provider selected while a local model stays resident, and
|
||||
// clearCheckpoint calls saveLastExternalCheckpoint(null), so clearing
|
||||
// unconditionally would delete a selection this button never touched.
|
||||
// Both spellings: status reports the concrete load path as the identifier
|
||||
// while the store may hold the advertised repo id, so matching only the
|
||||
// path leaves the store pinned to a model this button just freed.
|
||||
const store = useChatRuntimeStore.getState();
|
||||
const selected = store.params.checkpoint;
|
||||
const unloadedAliases = [checkpoint, status.active_model];
|
||||
if (
|
||||
selected &&
|
||||
!isExternalModelId(selected) &&
|
||||
unloadedAliases.some((alias) => modelIdsMatch(selected, alias))
|
||||
) {
|
||||
store.clearCheckpoint();
|
||||
}
|
||||
setUnloadError(null);
|
||||
refresh();
|
||||
} catch (err: unknown) {
|
||||
setUnloadError(
|
||||
err instanceof Error ? err.message : "Failed to unload the model",
|
||||
);
|
||||
} finally {
|
||||
setUnloading(false);
|
||||
}
|
||||
};
|
||||
const [statusFilter, setStatusFilter] = useState<MonitorStatusFilter>("all");
|
||||
const [query, setQuery] = useState("");
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null);
|
||||
|
||||
const visible = useMemo(
|
||||
() => filterEntries(entries, statusFilter, query),
|
||||
[entries, statusFilter, query],
|
||||
);
|
||||
const selected = useMemo(
|
||||
() => visible.find((entry) => entry.id === selectedId) ?? null,
|
||||
[visible, selectedId],
|
||||
);
|
||||
|
||||
// Refetch the selected entry while it streams so the payload grows with the reply.
|
||||
// Keyed on identity and revision, never on `details`: the fetch rewrites `details`
|
||||
// on every success, so depending on it loops on the detail endpoint.
|
||||
const selectedId_ = selected?.id ?? null;
|
||||
const selectedUpdatedAt = selected?.updated_at ?? null;
|
||||
const selectedIsMissing = selectedId_ != null && details[selectedId_] == null;
|
||||
const lastFetchedRef = useRef<string | null>(null);
|
||||
const attemptsRef = useRef<{ revision: string; count: number }>({
|
||||
revision: "",
|
||||
count: 0,
|
||||
});
|
||||
const [retryTick, setRetryTick] = useState(0);
|
||||
// Flips as a fetch settles either way, which is what lets a failure be noticed.
|
||||
const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_);
|
||||
useEffect(() => {
|
||||
if (selectedId_ == null || detailInFlight) {
|
||||
return;
|
||||
}
|
||||
// `updated_at` advances per poll while streaming and settles when terminal.
|
||||
// A missing payload always retries, covering a fetch that failed late.
|
||||
const revision = `${selectedId_}@${selectedUpdatedAt ?? ""}`;
|
||||
if (!selectedIsMissing && lastFetchedRef.current === revision) {
|
||||
return;
|
||||
}
|
||||
// A terminal row's revision never advances, so a failed fetch had nothing left to
|
||||
// re-run this effect. `loadingDetails` settling is the trigger; the count bounds
|
||||
// it, since the usual failure is an entry aged out of the ring buffer.
|
||||
if (attemptsRef.current.revision !== revision) {
|
||||
attemptsRef.current = { revision, count: 0 };
|
||||
}
|
||||
if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) {
|
||||
return;
|
||||
}
|
||||
attemptsRef.current.count += 1;
|
||||
// Only remember the revision when a fetch started: the in-flight guard can
|
||||
// refuse, and recording it anyway skips that revision for good.
|
||||
if (requestDetail(selectedId_)) {
|
||||
lastFetchedRef.current = revision;
|
||||
setRetryTick(0);
|
||||
} else {
|
||||
// Refused because an older fetch is running. No dep changes when it settles, so
|
||||
// without this nudge the rejected revision is never fetched.
|
||||
const timer = window.setTimeout(() => setRetryTick((n) => n + 1), 250);
|
||||
return () => window.clearTimeout(timer);
|
||||
}
|
||||
}, [
|
||||
selectedId_,
|
||||
selectedUpdatedAt,
|
||||
selectedIsMissing,
|
||||
requestDetail,
|
||||
retryTick,
|
||||
detailInFlight,
|
||||
]);
|
||||
|
||||
// The desktop webview's origin is tauri://, not the API server, and the packaged
|
||||
// app picks its port dynamically. Same source as the Agents tab.
|
||||
const origin = typeof window === "undefined" ? "" : window.location.origin;
|
||||
const baseUrl = `${isTauri ? (serverUrl ?? getApiBase()) : origin}/v1`;
|
||||
const serverStatus = data?.status ?? "idle";
|
||||
const statusCopy =
|
||||
serverStatus === "generating"
|
||||
? "Serving requests"
|
||||
: serverStatus === "ready"
|
||||
? "Ready"
|
||||
: "No model loaded";
|
||||
|
||||
return (
|
||||
<main className="mx-auto flex w-full max-w-6xl flex-col gap-6 px-6 pb-10 pt-12 font-heading sm:px-10">
|
||||
<header className="flex flex-wrap items-start justify-between gap-4">
|
||||
<div className="flex min-w-0 flex-col gap-1">
|
||||
<h1 className="text-ui-30 font-semibold leading-[1.04] tracking-[-0.028em] text-foreground sm:text-ui-34">
|
||||
API
|
||||
</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Live traffic through Unsloth's OpenAI-compatible server.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => setPaused(!paused)}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={paused ? PlayIcon : PauseIcon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
{paused ? "Resume" : "Pause"}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => void unloadActiveModel()}
|
||||
disabled={unloading || !data?.active_model}
|
||||
title={
|
||||
data?.active_model
|
||||
? `Unload ${data.active_model} and free its VRAM`
|
||||
: "No model is loaded"
|
||||
}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={PowerSocket01Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
{unloading ? "Unloading" : "Unload"}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={refresh}
|
||||
disabled={refreshing}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={RefreshIcon}
|
||||
strokeWidth={1.75}
|
||||
className={cn("size-4", refreshing && "animate-spin")}
|
||||
/>
|
||||
Refresh
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={entries.length === 0}
|
||||
onClick={() => {
|
||||
setSelectedId(null);
|
||||
void clear();
|
||||
}}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Delete02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
Clear log
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() =>
|
||||
useSettingsDialogStore.getState().openDialog("api-keys")
|
||||
}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Settings02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
API settings
|
||||
</Button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* What you check first when a client can't reach the API: the base URL
|
||||
to point it at, and what is loaded. */}
|
||||
<section className="flex flex-wrap items-center gap-x-6 gap-y-3 rounded-xl border border-border/60 bg-card px-4 py-3">
|
||||
<div className="flex min-w-0 items-center gap-2.5">
|
||||
<span className="flex size-8 shrink-0 items-center justify-center rounded-lg border border-border/60 bg-muted/40">
|
||||
<HugeiconsIcon
|
||||
icon={Globe02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
</span>
|
||||
<div className="flex min-w-0 flex-col">
|
||||
<span className="text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
Base URL
|
||||
</span>
|
||||
<span className="truncate font-mono text-ui-12 text-foreground">
|
||||
{baseUrl}
|
||||
</span>
|
||||
</div>
|
||||
<CopyButton value={baseUrl} label="Copy API base URL" />
|
||||
</div>
|
||||
<div className="flex min-w-0 flex-col">
|
||||
<span className="text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
Status
|
||||
</span>
|
||||
<span className="flex items-center gap-1.5 text-ui-12 text-foreground">
|
||||
<span
|
||||
className={cn(
|
||||
"size-2 rounded-full",
|
||||
serverStatus === "generating"
|
||||
? "bg-blue-500 animate-pulse"
|
||||
: serverStatus === "ready"
|
||||
? "bg-emerald-500"
|
||||
: "bg-muted-foreground",
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
{statusCopy}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 flex-1 flex-col">
|
||||
<span className="text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
Loaded model
|
||||
</span>
|
||||
<span className="truncate text-ui-12 text-foreground">
|
||||
{data?.active_model ?? "None"}
|
||||
{data?.context_length
|
||||
? ` · ${formatCount(data.context_length)} ctx`
|
||||
: ""}
|
||||
</span>
|
||||
</div>
|
||||
{paused ? (
|
||||
<span className="rounded-full border border-amber-500/40 bg-amber-500/10 px-2.5 py-1 text-ui-11 font-medium text-amber-700 dark:text-amber-500">
|
||||
Paused
|
||||
</span>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
{error || unloadError ? (
|
||||
<div className="rounded-xl border border-red-500/40 bg-red-500/5 px-4 py-3 text-sm text-red-600 dark:text-red-400">
|
||||
{error || unloadError}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<section className="grid grid-cols-2 gap-3 sm:grid-cols-3 lg:grid-cols-6">
|
||||
<StatCard
|
||||
label="In flight"
|
||||
value={formatCount(stats.active)}
|
||||
tone={stats.active > 0 ? "active" : "default"}
|
||||
/>
|
||||
<StatCard
|
||||
label="Requests"
|
||||
value={formatCount(stats.total)}
|
||||
hint="recent window"
|
||||
/>
|
||||
<StatCard label="Completed" value={formatCount(stats.completed)} />
|
||||
<StatCard
|
||||
label="Errors"
|
||||
value={formatCount(stats.errors)}
|
||||
tone={stats.errors > 0 ? "error" : "default"}
|
||||
hint={
|
||||
stats.errorRate != null
|
||||
? `${Math.round(stats.errorRate * 100)}% of finished`
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
<StatCard
|
||||
label="Avg latency"
|
||||
value={
|
||||
stats.avgDurationMs == null
|
||||
? "–"
|
||||
: formatDuration(stats.avgDurationMs)
|
||||
}
|
||||
hint={
|
||||
stats.maxDurationMs != null
|
||||
? `max ${formatDuration(stats.maxDurationMs)}`
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
<StatCard
|
||||
label="Throughput"
|
||||
value={
|
||||
stats.tokensPerSecond == null
|
||||
? "–"
|
||||
: `${stats.tokensPerSecond.toFixed(1)} tok/s`
|
||||
}
|
||||
hint={`${formatCount(stats.totalTokens)} tokens`}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section className="flex min-h-0 flex-col overflow-hidden rounded-xl border border-border/60 bg-card">
|
||||
<div className="flex flex-wrap items-center gap-2 border-b border-border/60 px-4 py-3">
|
||||
<Input
|
||||
value={query}
|
||||
onChange={(event) => setQuery(event.target.value)}
|
||||
placeholder="Search model, endpoint, preview or error"
|
||||
aria-label="Search API requests"
|
||||
className="h-9 w-full min-w-0 flex-1 rounded-full border-none bg-muted shadow-none dark:bg-background sm:w-64 sm:flex-none"
|
||||
/>
|
||||
<Select
|
||||
value={statusFilter}
|
||||
onValueChange={(value) =>
|
||||
setStatusFilter(value as MonitorStatusFilter)
|
||||
}
|
||||
>
|
||||
<SelectTrigger
|
||||
aria-label="Filter by status"
|
||||
className="h-9 w-[150px] rounded-full border-none bg-muted shadow-none dark:bg-background"
|
||||
>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{STATUS_FILTERS.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<span className="ml-auto shrink-0 text-ui-11 text-muted-foreground">
|
||||
{formatCount(visible.length)} of {formatCount(entries.length)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="grid min-h-0 grid-cols-1 lg:grid-cols-[minmax(0,380px)_minmax(0,1fr)]">
|
||||
<div className="max-h-[560px] min-h-[220px] overflow-y-auto border-b border-border/60 lg:border-b-0 lg:border-r">
|
||||
{loading ? (
|
||||
<div className="flex flex-col gap-3 p-4">
|
||||
{[0, 1, 2].map((i) => (
|
||||
<Skeleton key={i} className="h-16 w-full rounded-lg" />
|
||||
))}
|
||||
</div>
|
||||
) : visible.length === 0 ? (
|
||||
<p className="px-4 py-10 text-center text-sm text-muted-foreground">
|
||||
{entries.length === 0
|
||||
? "No API traffic yet. Point a client at the base URL above to see requests here."
|
||||
: "No requests match this filter."}
|
||||
</p>
|
||||
) : (
|
||||
visible.map((entry) => (
|
||||
<RequestRow
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
selected={entry.id === selectedId}
|
||||
onSelect={() => setSelectedId(entry.id)}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="max-h-[560px] min-h-[220px] overflow-y-auto">
|
||||
{selected ? (
|
||||
<RequestDetail
|
||||
entry={selected}
|
||||
detail={details[selected.id]}
|
||||
loading={loadingDetails.has(selected.id)}
|
||||
/>
|
||||
) : (
|
||||
<p className="flex h-full items-center justify-center px-6 py-10 text-center text-sm text-muted-foreground">
|
||||
Select a request to inspect its prompt, reply, tokens and
|
||||
errors.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<SavedModelSettingsPanel />
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// What a remote load will actually apply, otherwise unanswerable from outside the
|
||||
// process. Read only: the config also lives in the browser's per-model store, and
|
||||
// the model's settings page is the only place that owns both.
|
||||
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import {
|
||||
type ApiModelOverride,
|
||||
type ApiModelOverrides,
|
||||
fetchModelOverrides,
|
||||
} from "@/features/model-picker/api/model-overrides";
|
||||
import { type ReactElement, useCallback, useEffect, useState } from "react";
|
||||
|
||||
/** Summary of the fields the loader will apply, in load order. */
|
||||
function describeOverride(override: ApiModelOverride): string[] {
|
||||
const parts: string[] = [];
|
||||
if (override.custom_context_length) {
|
||||
parts.push(`${override.custom_context_length.toLocaleString()} context`);
|
||||
}
|
||||
if (override.max_seq_length) {
|
||||
parts.push(`${override.max_seq_length.toLocaleString()} max seq`);
|
||||
}
|
||||
if (override.kv_cache_dtype) {
|
||||
parts.push(`KV ${override.kv_cache_dtype}`);
|
||||
}
|
||||
if (override.speculative_type) {
|
||||
parts.push(
|
||||
override.spec_draft_n_max
|
||||
? `spec ${override.speculative_type} ×${override.spec_draft_n_max}`
|
||||
: `spec ${override.speculative_type}`,
|
||||
);
|
||||
}
|
||||
if (override.n_parallel) {
|
||||
parts.push(`${override.n_parallel} parallel slots`);
|
||||
}
|
||||
if (override.tensor_parallel) {
|
||||
parts.push("tensor parallel");
|
||||
}
|
||||
if (override.gpu_memory_mode === "manual") {
|
||||
parts.push("manual GPU memory");
|
||||
}
|
||||
if (override.gpu_layers != null) {
|
||||
parts.push(`${override.gpu_layers} GPU layers`);
|
||||
}
|
||||
if (override.n_cpu_moe) {
|
||||
parts.push(`${override.n_cpu_moe} MoE layers on CPU`);
|
||||
}
|
||||
if (override.gpu_ids?.length) {
|
||||
parts.push(`GPU ${override.gpu_ids.join(", ")}`);
|
||||
}
|
||||
if (override.chat_template_override) {
|
||||
parts.push("custom chat template");
|
||||
}
|
||||
if (override.llama_extra_args?.length) {
|
||||
parts.push(override.llama_extra_args.join(" "));
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
export function SavedModelSettingsPanel(): ReactElement {
|
||||
const [overrides, setOverrides] = useState<ApiModelOverrides | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
setOverrides(await fetchModelOverrides());
|
||||
setError(null);
|
||||
} catch (err: unknown) {
|
||||
setError(
|
||||
err instanceof Error
|
||||
? err.message
|
||||
: "Could not load saved model settings",
|
||||
);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
const entries = Object.entries(overrides ?? {});
|
||||
|
||||
return (
|
||||
<section className="flex flex-col gap-3">
|
||||
<div className="flex flex-col gap-1">
|
||||
<h2 className="text-ui-16 font-semibold tracking-[-0.01em] text-foreground">
|
||||
Settings applied on API load
|
||||
</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
When a request names one of these models, Unsloth loads it with these
|
||||
settings, the same ones you saved in the model's settings page.
|
||||
Models without an entry load with app defaults. Edit or forget an
|
||||
entry from that model's settings, which keeps this list and the
|
||||
picker in step.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error ? (
|
||||
<div className="rounded-xl border border-red-500/40 bg-red-500/5 px-4 py-3 text-sm text-red-600 dark:text-red-400">
|
||||
{error}
|
||||
</div>
|
||||
) : overrides == null ? (
|
||||
<div className="flex flex-col gap-2">
|
||||
{[0, 1].map((i) => (
|
||||
<Skeleton key={i} className="h-14 w-full rounded-xl" />
|
||||
))}
|
||||
</div>
|
||||
) : entries.length === 0 ? (
|
||||
<p className="rounded-xl border border-border/60 bg-card px-4 py-6 text-center text-sm text-muted-foreground">
|
||||
No saved model settings yet. Open a model's settings, turn on
|
||||
"Remember for this model", and it will be applied to API
|
||||
loads too.
|
||||
</p>
|
||||
) : (
|
||||
<ul className="flex flex-col gap-2">
|
||||
{entries.map(([modelId, override]) => {
|
||||
const summary = describeOverride(override);
|
||||
return (
|
||||
<li
|
||||
key={modelId}
|
||||
className="flex min-w-0 items-start gap-3 rounded-xl border border-border/60 bg-card px-4 py-3"
|
||||
>
|
||||
<div className="flex min-w-0 flex-1 flex-col gap-1">
|
||||
<span className="min-w-0 break-all font-mono text-ui-12 font-medium text-foreground">
|
||||
{modelId}
|
||||
</span>
|
||||
<span className="min-w-0 break-words text-ui-11 text-muted-foreground">
|
||||
{summary.length > 0 ? summary.join(" · ") : "App defaults"}
|
||||
</span>
|
||||
</div>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
13
studio/frontend/src/features/api-monitor/index.ts
Normal file
13
studio/frontend/src/features/api-monitor/index.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
export { ApiMonitorPage } from "./api-monitor-page";
|
||||
export { ApiMonitorOverlay } from "./api-monitor-overlay";
|
||||
export { useApiMonitorOverlayStore } from "./overlay-store";
|
||||
export {
|
||||
computeStats,
|
||||
filterEntries,
|
||||
useApiMonitor,
|
||||
type MonitorStats,
|
||||
type MonitorStatusFilter,
|
||||
} from "./use-api-monitor";
|
||||
44
studio/frontend/src/features/api-monitor/lifecycle.ts
Normal file
44
studio/frontend/src/features/api-monitor/lifecycle.ts
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Labels for model load/unload/download rows, shared by the overlay and the page.
|
||||
//
|
||||
// Their own module because the overlay is mounted from __root.tsx: importing them
|
||||
// from the page would pull the whole page into the eager bundle and undo the
|
||||
// route's lazyRouteComponent.
|
||||
|
||||
import type { ApiMonitorEntry } from "@/features/chat/types/api";
|
||||
|
||||
// A lifecycle row is a model load/unload/download, not an HTTP call: it carries an
|
||||
// event and reason instead of a prompt, so there is no payload to expand.
|
||||
export function isLifecycleEntry(entry: ApiMonitorEntry): boolean {
|
||||
return entry.kind === "lifecycle";
|
||||
}
|
||||
|
||||
export function lifecycleLabel(entry: ApiMonitorEntry): string {
|
||||
if (entry.event === "unload") {
|
||||
return entry.reason === "idle" ? "Model unloaded (idle)" : "Model unloaded";
|
||||
}
|
||||
if (entry.event === "download") {
|
||||
if (entry.status === "running") {
|
||||
const pct = entry.progress;
|
||||
return typeof pct === "number"
|
||||
? `Downloading model (${Math.round(pct)}%)`
|
||||
: "Downloading model";
|
||||
}
|
||||
if (entry.status === "completed") {
|
||||
return "Model downloaded";
|
||||
}
|
||||
// A cancel is deliberate, so calling it a failure misreads the user's action.
|
||||
return entry.status === "cancelled"
|
||||
? "Model download cancelled"
|
||||
: "Model download failed";
|
||||
}
|
||||
if (entry.status === "running") {
|
||||
return "Loading model";
|
||||
}
|
||||
if (entry.status === "completed") {
|
||||
return "Model loaded";
|
||||
}
|
||||
return "Model load failed";
|
||||
}
|
||||
131
studio/frontend/src/features/api-monitor/new-traffic.ts
Normal file
131
studio/frontend/src/features/api-monitor/new-traffic.ts
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Which rows of a monitor snapshot are traffic this session has not shown yet.
|
||||
// Split out of the overlay so it can be driven without a browser.
|
||||
|
||||
import type { ApiMonitorEntry } from "@/features/chat/types/api";
|
||||
|
||||
export type WatchedEntry = Pick<
|
||||
ApiMonitorEntry,
|
||||
"id" | "status" | "via_api_key" | "started_at"
|
||||
>;
|
||||
|
||||
export interface WatchedResponse {
|
||||
entries: readonly WatchedEntry[];
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
server_time?: number | null;
|
||||
}
|
||||
|
||||
export interface ApiMonitorWatch {
|
||||
/** The first snapshot has been folded in and its backlog written off. */
|
||||
seeded: boolean;
|
||||
/** Ids already shown. A set, not "the newest id": finishing moves an entry to
|
||||
* the front, so the head flips without any new traffic. */
|
||||
seenIds: Set<string>;
|
||||
/** performance.now() when this watch began; monotonic, so a client clock step
|
||||
* mid-session cannot move it. */
|
||||
watchStartedAt: number;
|
||||
/** This seed follows a stay on the full page rather than starting a session. */
|
||||
resumed: boolean;
|
||||
}
|
||||
|
||||
export function createWatch(nowMs: number): ApiMonitorWatch {
|
||||
return {
|
||||
seeded: false,
|
||||
seenIds: new Set(),
|
||||
watchStartedAt: nowMs,
|
||||
resumed: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-anchor as the poll stands up, and only while still unseeded: the first
|
||||
* snapshot can land long after the overlay mounted (a hidden tab issues no
|
||||
* fetch, a backend still coming up fails one), and dating the backlog from
|
||||
* mount would write that whole gap off as history.
|
||||
*/
|
||||
export function startWatching(watch: ApiMonitorWatch, nowMs: number): void {
|
||||
if (!watch.seeded) {
|
||||
watch.watchStartedAt = nowMs;
|
||||
}
|
||||
}
|
||||
|
||||
/** The full page took over; what it showed is not new traffic on the way back. */
|
||||
export function rearmWatch(watch: ApiMonitorWatch): void {
|
||||
watch.seeded = false;
|
||||
watch.resumed = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* When this watch began, on the server's clock.
|
||||
*
|
||||
* The server's own ``time.time()`` minus a browser *duration*, never minus a
|
||||
* browser timestamp, so a browser clock that disagrees with the server's -- a
|
||||
* Studio behind a tunnel, in a container, on a host that has not run NTP --
|
||||
* cancels instead of skewing the answer. Null on a backend with no clock field,
|
||||
* which keeps the old behaviour.
|
||||
*/
|
||||
function historyCutoff(
|
||||
watch: ApiMonitorWatch,
|
||||
response: WatchedResponse,
|
||||
nowMs: number,
|
||||
): number | null {
|
||||
const serverTime = response.server_time;
|
||||
if (typeof serverTime !== "number" || !Number.isFinite(serverTime)) {
|
||||
return null;
|
||||
}
|
||||
return serverTime - Math.max(0, nowMs - watch.watchStartedAt) / 1000;
|
||||
}
|
||||
|
||||
function isHistory(entry: WatchedEntry, cutoff: number | null): boolean {
|
||||
// Still running at the first snapshot: it started while Studio was loading, so
|
||||
// it is unseen live traffic.
|
||||
if (entry.status === "running") {
|
||||
return false;
|
||||
}
|
||||
if (cutoff == null || !Number.isFinite(entry.started_at)) {
|
||||
return true;
|
||||
}
|
||||
// Finished before the first snapshot is not the same as started before we did:
|
||||
// a call made while the tab was hidden is already terminal when the poll
|
||||
// finally runs, and writing it off is how the panel misses the first request.
|
||||
return entry.started_at <= cutoff;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a snapshot in and report whether it holds API-key traffic not shown yet.
|
||||
*/
|
||||
export function observeResponse(
|
||||
watch: ApiMonitorWatch,
|
||||
response: WatchedResponse,
|
||||
nowMs: number,
|
||||
): boolean {
|
||||
const { entries } = response;
|
||||
if (!watch.seeded) {
|
||||
watch.seeded = true;
|
||||
const { resumed } = watch;
|
||||
watch.resumed = false;
|
||||
const cutoff = historyCutoff(watch, response, nowMs);
|
||||
// A rearm is not a fresh watch. isHistory holds a running row back on
|
||||
// purpose: at a session's first snapshot it started while Studio was still
|
||||
// loading and nobody has seen it. On the way off the full page the opposite
|
||||
// is true -- that page was showing this same feed, running rows included --
|
||||
// so seeding from isHistory alone reopens the overlay on the request the
|
||||
// user was reading when they left. Everything the page could show is read.
|
||||
watch.seenIds = new Set(
|
||||
entries
|
||||
.filter((entry) => resumed || isHistory(entry, cutoff))
|
||||
.map((e) => e.id),
|
||||
);
|
||||
}
|
||||
const seen = watch.seenIds;
|
||||
// Only API-key traffic counts: Studio's own chat uses these same endpoints, and
|
||||
// this panel is about serving other clients.
|
||||
const hasNewTraffic = entries.some(
|
||||
(entry) => entry.via_api_key && !seen.has(entry.id),
|
||||
);
|
||||
// Re-seed each poll so the set stays bounded by the server's ring buffer.
|
||||
watch.seenIds = new Set(entries.map((entry) => entry.id));
|
||||
return hasNewTraffic;
|
||||
}
|
||||
78
studio/frontend/src/features/api-monitor/overlay-store.ts
Normal file
78
studio/frontend/src/features/api-monitor/overlay-store.ts
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { create } from "zustand";
|
||||
import { createJSONStorage, persist } from "zustand/middleware";
|
||||
|
||||
/**
|
||||
* localStorage that cannot throw: Safari private browsing, blocked cookies and an
|
||||
* opaque webview origin all make `window.localStorage` throw on access. Losing the
|
||||
* preference there is fine; taking the whole panel down with it is not.
|
||||
*/
|
||||
const safeStorage = {
|
||||
getItem: (name: string): string | null => {
|
||||
try {
|
||||
return window.localStorage.getItem(name);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
},
|
||||
setItem: (name: string, value: string): void => {
|
||||
try {
|
||||
window.localStorage.setItem(name, value);
|
||||
} catch {
|
||||
// Quota exceeded or denied; the preference stays session-only.
|
||||
}
|
||||
},
|
||||
removeItem: (name: string): void => {
|
||||
try {
|
||||
window.localStorage.removeItem(name);
|
||||
} catch {
|
||||
// Same.
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
interface ApiMonitorOverlayState {
|
||||
/** Whether the floating panel is on screen right now. Session state. */
|
||||
isOpen: boolean;
|
||||
/** Set on close so the panel does not pop back in the same burst. */
|
||||
suppressed: boolean;
|
||||
/** Persisted opt out: when false the panel never opens itself. */
|
||||
autoOpen: boolean;
|
||||
open: () => void;
|
||||
close: () => void;
|
||||
setAutoOpen: (autoOpen: boolean) => void;
|
||||
}
|
||||
|
||||
/** Only `autoOpen` persists; a dismissal lasts the sitting, not forever. */
|
||||
export const useApiMonitorOverlayStore = create<ApiMonitorOverlayState>()(
|
||||
persist(
|
||||
(set) => ({
|
||||
isOpen: false,
|
||||
suppressed: false,
|
||||
autoOpen: true,
|
||||
open: () => set({ isOpen: true, suppressed: false }),
|
||||
close: () => set({ isOpen: false, suppressed: true }),
|
||||
setAutoOpen: (autoOpen) => set({ autoOpen }),
|
||||
}),
|
||||
{
|
||||
name: "unsloth_api_monitor_overlay",
|
||||
version: 1,
|
||||
storage: createJSONStorage(() => safeStorage),
|
||||
partialize: (state) => ({ autoOpen: state.autoOpen }),
|
||||
// Without this a version bump discards the payload and hands the popup back
|
||||
// to someone who had turned it off.
|
||||
migrate: (persisted) => persisted,
|
||||
// Explicit merge so an older stored payload cannot resurrect `isOpen`.
|
||||
merge: (persisted, current) => ({
|
||||
...current,
|
||||
autoOpen:
|
||||
typeof (persisted as { autoOpen?: unknown } | null)?.autoOpen ===
|
||||
"boolean"
|
||||
? (persisted as { autoOpen: boolean }).autoOpen
|
||||
: current.autoOpen,
|
||||
}),
|
||||
},
|
||||
),
|
||||
);
|
||||
313
studio/frontend/src/features/api-monitor/use-api-monitor.ts
Normal file
313
studio/frontend/src/features/api-monitor/use-api-monitor.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import {
|
||||
clearApiMonitor,
|
||||
getApiMonitor,
|
||||
getApiMonitorEntry,
|
||||
} from "@/features/chat/api/chat-api";
|
||||
import type {
|
||||
ApiMonitorEntry,
|
||||
ApiMonitorResponse,
|
||||
} from "@/features/chat/types/api";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
/** Poll cadence while live. Matches the settings console it replaces. */
|
||||
const POLL_INTERVAL_MS = 1500;
|
||||
|
||||
export type MonitorStatusFilter =
|
||||
| "all"
|
||||
| "running"
|
||||
| "completed"
|
||||
| "error"
|
||||
| "cancelled";
|
||||
|
||||
export interface MonitorStats {
|
||||
active: number;
|
||||
total: number;
|
||||
completed: number;
|
||||
errors: number;
|
||||
cancelled: number;
|
||||
/** Mean duration over finished requests, or null when none have finished. */
|
||||
avgDurationMs: number | null;
|
||||
/** Slowest finished request, for spotting a pathological call. */
|
||||
maxDurationMs: number | null;
|
||||
totalTokens: number;
|
||||
/** Share of finished requests that failed, 0-1. Null when nothing finished. */
|
||||
errorRate: number | null;
|
||||
/** Mean completion tokens per second over requests that reported both. */
|
||||
tokensPerSecond: number | null;
|
||||
}
|
||||
|
||||
function isTerminal(entry: ApiMonitorEntry): boolean {
|
||||
return entry.status !== "running";
|
||||
}
|
||||
|
||||
function completionTokens(entry: ApiMonitorEntry): number | null {
|
||||
if (entry.completion_tokens != null) {
|
||||
return entry.completion_tokens;
|
||||
}
|
||||
// Some providers report only a total, so subtracting the prompt is the best
|
||||
// estimate of what was generated.
|
||||
if (entry.total_tokens != null && entry.prompt_tokens != null) {
|
||||
return Math.max(0, entry.total_tokens - entry.prompt_tokens);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function entryTokens(entry: ApiMonitorEntry): number {
|
||||
if (entry.total_tokens != null) {
|
||||
return entry.total_tokens;
|
||||
}
|
||||
return (entry.prompt_tokens ?? 0) + (entry.completion_tokens ?? 0);
|
||||
}
|
||||
|
||||
export function computeStats(entries: ApiMonitorEntry[]): MonitorStats {
|
||||
let active = 0;
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
let cancelled = 0;
|
||||
let totalTokens = 0;
|
||||
let durationSum = 0;
|
||||
let durationCount = 0;
|
||||
let maxDurationMs: number | null = null;
|
||||
// Total tokens over total time, not the mean of each request's rate: averaging
|
||||
// rates lets one tiny fast request outweigh a long slow one.
|
||||
let generatedTokens = 0;
|
||||
let generatedDurationMs = 0;
|
||||
|
||||
let requests = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
// A load, unload or download is not an HTTP call. It reads as "running" for the
|
||||
// whole load, so counting it reports an in-flight request with no client waiting
|
||||
// and folds a multi-minute download into "Avg latency". The backend leaves these
|
||||
// out of active_count too, so counting them would also disagree with the API.
|
||||
if (entry.kind === "lifecycle") {
|
||||
continue;
|
||||
}
|
||||
requests += 1;
|
||||
totalTokens += entryTokens(entry);
|
||||
if (entry.status === "running") {
|
||||
active += 1;
|
||||
} else if (entry.status === "error") {
|
||||
errors += 1;
|
||||
} else if (entry.status === "cancelled") {
|
||||
cancelled += 1;
|
||||
} else {
|
||||
completed += 1;
|
||||
}
|
||||
const duration = entry.duration_ms;
|
||||
if (duration != null && isTerminal(entry)) {
|
||||
durationSum += duration;
|
||||
durationCount += 1;
|
||||
maxDurationMs =
|
||||
maxDurationMs == null ? duration : Math.max(maxDurationMs, duration);
|
||||
const generated = completionTokens(entry);
|
||||
// A sub-millisecond duration divides into a meaningless rate.
|
||||
if (generated != null && generated > 0 && duration > 0) {
|
||||
generatedTokens += generated;
|
||||
generatedDurationMs += duration;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const finished = completed + errors + cancelled;
|
||||
return {
|
||||
active,
|
||||
total: requests,
|
||||
completed,
|
||||
errors,
|
||||
cancelled,
|
||||
avgDurationMs: durationCount > 0 ? durationSum / durationCount : null,
|
||||
maxDurationMs,
|
||||
totalTokens,
|
||||
errorRate: finished > 0 ? errors / finished : null,
|
||||
tokensPerSecond:
|
||||
generatedDurationMs > 0
|
||||
? generatedTokens / (generatedDurationMs / 1000)
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function filterEntries(
|
||||
entries: ApiMonitorEntry[],
|
||||
status: MonitorStatusFilter,
|
||||
query: string,
|
||||
): ApiMonitorEntry[] {
|
||||
const needle = query.trim().toLowerCase();
|
||||
return entries.filter((entry) => {
|
||||
if (status !== "all" && entry.status !== status) {
|
||||
return false;
|
||||
}
|
||||
if (!needle) {
|
||||
return true;
|
||||
}
|
||||
// The fields a debugging session keys off: model, endpoint, and the previews and
|
||||
// error text visible in the row. Coerced, not trusted: these arrive over the
|
||||
// network, and one malformed entry throwing here would blank the whole log.
|
||||
return [
|
||||
entry.model,
|
||||
entry.endpoint,
|
||||
entry.prompt_preview,
|
||||
entry.reply_preview,
|
||||
entry.error,
|
||||
].some((field) =>
|
||||
String(field ?? "")
|
||||
.toLowerCase()
|
||||
.includes(needle),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
interface UseApiMonitorResult {
|
||||
data: ApiMonitorResponse | null;
|
||||
entries: ApiMonitorEntry[];
|
||||
stats: MonitorStats;
|
||||
error: string | null;
|
||||
/** True until the first response lands, so skeletons show once. */
|
||||
loading: boolean;
|
||||
refreshing: boolean;
|
||||
paused: boolean;
|
||||
setPaused: (paused: boolean) => void;
|
||||
refresh: () => void;
|
||||
clear: () => Promise<void>;
|
||||
/** Full prompt/reply for expanded entries, keyed by entry id. */
|
||||
details: Record<string, ApiMonitorEntry>;
|
||||
loadingDetails: ReadonlySet<string>;
|
||||
requestDetail: (id: string) => boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Live view of the server's OpenAI-compatible API traffic.
|
||||
*
|
||||
* Polls rather than streams because the backing monitor is an in-memory ring buffer
|
||||
* with no change feed. Polling self-reschedules (never overlapping), and pausing
|
||||
* stops it so reading a stalled payload is not fighting a list that reorders.
|
||||
*
|
||||
* `intervalMs` lets a caller trade freshness for cost: the full page wants the
|
||||
* default live cadence, while the floating overlay slows right down when it is
|
||||
* closed and only watching for the traffic that should pop it open.
|
||||
*/
|
||||
export function useApiMonitor({
|
||||
intervalMs = POLL_INTERVAL_MS,
|
||||
}: { intervalMs?: number } = {}): UseApiMonitorResult {
|
||||
const [data, setData] = useState<ApiMonitorResponse | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [paused, setPaused] = useState(false);
|
||||
const [details, setDetails] = useState<Record<string, ApiMonitorEntry>>({});
|
||||
const [loadingDetails, setLoadingDetails] = useState<Set<string>>(
|
||||
() => new Set(),
|
||||
);
|
||||
// Mirrors `loadingDetails` outside React state so the fetch guard sees same-tick
|
||||
// writes; async state updates would let duplicates through.
|
||||
const inFlightDetails = useRef<Set<string>>(new Set());
|
||||
|
||||
const load = useCallback(async (): Promise<void> => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
const next = await getApiMonitor();
|
||||
setData(next);
|
||||
setError(null);
|
||||
} catch (err: unknown) {
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (paused) {
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
let timer: number | undefined;
|
||||
|
||||
function poll(): void {
|
||||
getApiMonitor()
|
||||
.then((next) => {
|
||||
if (cancelled) return;
|
||||
setData(next);
|
||||
setError(null);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (cancelled) return;
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
})
|
||||
.finally(() => {
|
||||
if (cancelled) return;
|
||||
setLoading(false);
|
||||
timer = window.setTimeout(poll, intervalMs);
|
||||
});
|
||||
}
|
||||
|
||||
poll();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer !== undefined) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
}, [paused, intervalMs]);
|
||||
|
||||
// Returns whether a fetch started: a caller must not record "fetched revision N"
|
||||
// when the guard refused, or that revision is skipped once updated_at settles.
|
||||
const requestDetail = useCallback((id: string): boolean => {
|
||||
if (inFlightDetails.current.has(id)) {
|
||||
return false;
|
||||
}
|
||||
inFlightDetails.current.add(id);
|
||||
setLoadingDetails((prev) => new Set(prev).add(id));
|
||||
getApiMonitorEntry(id)
|
||||
.then((entry) => {
|
||||
setDetails((prev) => ({ ...prev, [id]: entry }));
|
||||
})
|
||||
.catch(() => {
|
||||
// Aged out of the ring buffer; drop the stale copy so the UI falls back to
|
||||
// the row previews instead of a frozen payload.
|
||||
setDetails((prev) => {
|
||||
if (!(id in prev)) return prev;
|
||||
const next = { ...prev };
|
||||
delete next[id];
|
||||
return next;
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
inFlightDetails.current.delete(id);
|
||||
setLoadingDetails((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(id);
|
||||
return next;
|
||||
});
|
||||
});
|
||||
return true;
|
||||
}, []);
|
||||
|
||||
const clear = useCallback(async (): Promise<void> => {
|
||||
await clearApiMonitor();
|
||||
setDetails({});
|
||||
await load();
|
||||
}, [load]);
|
||||
|
||||
const entries = useMemo(() => data?.entries ?? [], [data]);
|
||||
const stats = useMemo(() => computeStats(entries), [entries]);
|
||||
|
||||
return {
|
||||
data,
|
||||
entries,
|
||||
stats,
|
||||
error,
|
||||
loading,
|
||||
refreshing,
|
||||
paused,
|
||||
setPaused,
|
||||
refresh: () => void load(),
|
||||
clear,
|
||||
details,
|
||||
loadingDetails,
|
||||
requestDetail,
|
||||
};
|
||||
}
|
||||
|
|
@ -129,6 +129,13 @@ export async function getApiMonitorEntry(id: string): Promise<ApiMonitorEntry> {
|
|||
return parseJsonOrThrow<ApiMonitorEntry>(response);
|
||||
}
|
||||
|
||||
export async function clearApiMonitor(): Promise<void> {
|
||||
const response = await authFetch("/api/inference/monitor", {
|
||||
method: "DELETE",
|
||||
});
|
||||
await parseJsonOrThrow<{ cleared: boolean }>(response);
|
||||
}
|
||||
|
||||
export interface ActiveGenerationsResponse {
|
||||
count: number;
|
||||
/** Conversations with a generation in flight. Shorter than `count` when a
|
||||
|
|
|
|||
|
|
@ -35,6 +35,10 @@ export type {
|
|||
GgufVariantDetail,
|
||||
InferenceStatusResponse,
|
||||
} from "./types/api";
|
||||
export {
|
||||
applyActiveModelStatusToStore,
|
||||
resolveInferenceCheckpointId,
|
||||
} from "./lib/apply-inference-status-to-store";
|
||||
export {
|
||||
ChatSettingsPanel,
|
||||
ParamSlider,
|
||||
|
|
|
|||
|
|
@ -301,6 +301,9 @@ export interface ApiMonitorEntry {
|
|||
model: string;
|
||||
prompt?: string;
|
||||
reply?: string;
|
||||
// True for API-key callers, not UI sessions. The floating panel keys its
|
||||
// auto-open off this so Studio's own chat does not pop it.
|
||||
via_api_key: boolean;
|
||||
prompt_preview: string;
|
||||
reply_preview: string;
|
||||
prompt_truncated: boolean;
|
||||
|
|
@ -326,6 +329,10 @@ export interface ApiMonitorEntry {
|
|||
|
||||
export interface ApiMonitorResponse {
|
||||
status: "idle" | "ready" | "generating";
|
||||
// Server wall clock (seconds) when the snapshot was taken, so an entry's
|
||||
// started_at can be dated without trusting the browser's clock to agree.
|
||||
// Absent on a backend older than the field.
|
||||
server_time?: number;
|
||||
active_model?: string | null;
|
||||
context_length?: number | null;
|
||||
active_requests: number;
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import {
|
|||
ArrowReloadHorizontalIcon,
|
||||
Delete02Icon,
|
||||
Download01Icon,
|
||||
Settings02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import {
|
||||
|
|
@ -82,6 +83,39 @@ export function CardDivider() {
|
|||
);
|
||||
}
|
||||
|
||||
export function CardSettingsButton({
|
||||
label,
|
||||
onClick,
|
||||
}: {
|
||||
label: string;
|
||||
onClick: () => void;
|
||||
}) {
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
aria-label={label}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onClick();
|
||||
}}
|
||||
className="inline-flex size-7 shrink-0 cursor-pointer items-center justify-center rounded-full text-muted-foreground opacity-0 transition-[opacity,background-color,color] duration-150 hover:bg-foreground/[0.06] hover:text-foreground focus-visible:opacity-100 group-hover/dl:opacity-100 dark:hover:bg-white/[0.08]"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Settings02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top" className="tooltip-compact">
|
||||
{label}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
export function CardDeleteButton({
|
||||
label,
|
||||
onClick,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,154 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Full-page settings for one model, opened from the Hub.
|
||||
//
|
||||
// The same controls exist in the chat picker's popover, but a popover is a poor
|
||||
// place to work through every knob. This gives them a page and says plainly that
|
||||
// what is saved here is what an API load uses, mirrored by ModelConfigPage.
|
||||
|
||||
import {
|
||||
ModelConfigPage,
|
||||
type ModelPickTarget,
|
||||
modelConfigInstanceKey,
|
||||
} from "@/features/model-picker";
|
||||
import type { PerModelConfig } from "@/features/model-picker";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
export function HubModelSettingsView({
|
||||
target,
|
||||
loadedConfig = null,
|
||||
loadedContextLength = null,
|
||||
onBack,
|
||||
onRun,
|
||||
compact = false,
|
||||
}: {
|
||||
target: ModelPickTarget;
|
||||
/** Non-null when this model is loaded, so the page can show live values. */
|
||||
loadedConfig?: PerModelConfig | null;
|
||||
loadedContextLength?: number | null;
|
||||
onBack: () => void;
|
||||
/** Apply + load with these settings. */
|
||||
onRun: (config: PerModelConfig) => void;
|
||||
compact?: boolean;
|
||||
}) {
|
||||
const scrollRef = useRef<HTMLDivElement | null>(null);
|
||||
const [scrolled, setScrolled] = useState(false);
|
||||
// Mirrors HubDetailView so this view sits at the Hub's measure.
|
||||
const measure = compact
|
||||
? "mx-auto w-full max-w-[860px] px-5 sm:px-5"
|
||||
: "mx-auto w-full max-w-[1100px] px-5 sm:px-8";
|
||||
|
||||
useEffect(() => {
|
||||
const el = scrollRef.current;
|
||||
if (!el) return;
|
||||
const onScroll = () => {
|
||||
const next = el.scrollTop > 0;
|
||||
setScrolled((current) => (current === next ? current : next));
|
||||
};
|
||||
onScroll();
|
||||
el.addEventListener("scroll", onScroll, { passive: true });
|
||||
return () => el.removeEventListener("scroll", onScroll);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="relative flex min-h-0 flex-1 flex-col">
|
||||
<div
|
||||
aria-hidden="true"
|
||||
data-scrolled={scrolled || undefined}
|
||||
className="hub-scroll-fade pointer-events-none absolute inset-x-0 top-0 z-10 h-7"
|
||||
/>
|
||||
<div
|
||||
ref={scrollRef}
|
||||
data-hub-scroll="true"
|
||||
className={cn(
|
||||
"min-h-0 flex-1 overflow-x-hidden overflow-y-auto [overflow-anchor:none] [scrollbar-width:thin]",
|
||||
compact
|
||||
? "mr-2 [scrollbar-gutter:stable]"
|
||||
: "[scrollbar-gutter:stable_both-edges]",
|
||||
)}
|
||||
>
|
||||
<div
|
||||
className="hub-detail-bar sticky top-0 z-20"
|
||||
data-scrolled={scrolled || undefined}
|
||||
>
|
||||
<div className={`${measure} py-3`}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onBack}
|
||||
className="-ml-1.5 inline-flex h-8 cursor-pointer select-none items-center gap-1.5 rounded-full pl-1.5 pr-2.5 text-ui-12p5 font-medium text-muted-foreground transition-colors hover:bg-foreground/[0.05] hover:text-foreground dark:hover:bg-white/[0.06]"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={ArrowLeft01Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-3.5"
|
||||
/>
|
||||
Back to Hub
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className={cn(measure, "pb-20")}>
|
||||
<header className="flex flex-col gap-1 pb-5">
|
||||
<h1 className="min-w-0 break-words text-ui-24 font-semibold leading-[1.1] tracking-[-0.022em] text-foreground">
|
||||
{target.displayName}
|
||||
</h1>
|
||||
<p className="min-w-0 break-all text-ui-12 text-muted-foreground">
|
||||
{target.id}
|
||||
{target.ggufVariant ? ` · ${target.ggufVariant}` : ""}
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<div className="mb-5 flex items-start gap-2.5 rounded-xl border border-border/60 bg-card px-4 py-3">
|
||||
<span className="mt-0.5 flex size-7 shrink-0 items-center justify-center rounded-lg border border-border/60 bg-muted/40">
|
||||
<HugeiconsIcon
|
||||
icon={Globe02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-3.5"
|
||||
/>
|
||||
</span>
|
||||
<p className="min-w-0 text-ui-12 leading-[1.5] text-muted-foreground">
|
||||
{/* Only what auto-switch can reach is mirrored: it indexes GGUFs and
|
||||
skips Ollama, so anything else cannot be loaded by the API. */}
|
||||
{(target.apiLoadable ?? target.isGguf)
|
||||
? "Saved settings apply everywhere this model loads, including when an OpenAI-compatible API request asks for it."
|
||||
: "Saved settings apply everywhere Studio loads this model."}{" "}
|
||||
Turn on{" "}
|
||||
<span className="font-medium text-foreground">
|
||||
Remember for this model
|
||||
</span>{" "}
|
||||
below to keep them.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="rounded-xl border border-border/60 bg-card px-4 py-4">
|
||||
<ModelConfigPage
|
||||
// Keyed on the live config too, like the sidebar entry.
|
||||
// ModelConfigPage reads loadedConfig once, in its useState
|
||||
// initializer, so opening this page before /api/inference/status
|
||||
// has hydrated (or while the target is still loading) would
|
||||
// otherwise leave the editor on saved/default values for a model
|
||||
// that is running with something else, and Apply would write them
|
||||
// back over it.
|
||||
key={modelConfigInstanceKey(
|
||||
target.id,
|
||||
target.ggufVariant,
|
||||
loadedConfig,
|
||||
)}
|
||||
target={target}
|
||||
onRun={onRun}
|
||||
loadedConfig={loadedConfig}
|
||||
loadedContextLength={loadedContextLength}
|
||||
variant="page"
|
||||
// The page heading already names the model; "Run settings" would repeat it.
|
||||
showHeader={false}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -58,6 +58,7 @@ import { useHfTokenStore } from "../stores/hf-token-store";
|
|||
import { DotTag } from "./dot-tag";
|
||||
import {
|
||||
CardDeleteButton,
|
||||
CardSettingsButton,
|
||||
CardUpdateButton,
|
||||
DeleteConfirmDialog,
|
||||
UpdateConfirmDialog,
|
||||
|
|
@ -104,6 +105,14 @@ interface LocalOnDeviceCardProps {
|
|||
onEject?: () => void;
|
||||
onTrain?: () => void;
|
||||
onChange?: () => void;
|
||||
/**
|
||||
* Open settings for the quant this card is showing.
|
||||
*
|
||||
* ``quantIsUserPicked`` says whether that quant came from a pick in this
|
||||
* card's selector or was derived from the resident model, which decides
|
||||
* whether a fresher status read may override it.
|
||||
*/
|
||||
onOpenSettings?: (ggufVariant: string | null, quantIsUserPicked: boolean) => void;
|
||||
}
|
||||
|
||||
function formatAdapterLabel(
|
||||
|
|
@ -225,6 +234,7 @@ export function LocalOnDeviceCard({
|
|||
onEject,
|
||||
onTrain,
|
||||
onChange,
|
||||
onOpenSettings,
|
||||
}: LocalOnDeviceCardProps) {
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
const [updateOpen, setUpdateOpen] = useState(false);
|
||||
|
|
@ -367,6 +377,14 @@ export function LocalOnDeviceCard({
|
|||
)?.quant ??
|
||||
sortedVariants?.[0]?.quant ??
|
||||
null);
|
||||
// Only the first branch above is a choice; the rest are fallbacks that read the
|
||||
// store, and the store can be stale for as long as this window keeps focus.
|
||||
const quantIsUserPicked = Boolean(
|
||||
selectedVariantOverride &&
|
||||
sortedVariants?.some((variant) =>
|
||||
ggufVariantsMatch(variant.quant, selectedVariantOverride),
|
||||
),
|
||||
);
|
||||
const selectedVariant =
|
||||
sortedVariants?.find((variant) =>
|
||||
ggufVariantsMatch(variant.quant, selectedQuant),
|
||||
|
|
@ -578,6 +596,16 @@ export function LocalOnDeviceCard({
|
|||
)}
|
||||
</span>
|
||||
<div className="ml-auto flex items-center gap-0.5">
|
||||
{onOpenSettings && (
|
||||
<CardSettingsButton
|
||||
label={`Settings for ${repoId}`}
|
||||
// Pass the quant this card resolved, so the settings page edits the
|
||||
// variant on screen rather than the repo.
|
||||
onClick={() =>
|
||||
onOpenSettings(selectedQuant ?? null, quantIsUserPicked)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{canUpdate && (
|
||||
<CardUpdateButton
|
||||
label={`Update ${repoId}`}
|
||||
|
|
|
|||
|
|
@ -403,6 +403,8 @@ export type ModelInspectorActions = {
|
|||
onTrain?: () => void;
|
||||
onInventoryChange?: () => void;
|
||||
onSearchHub?: (query: string) => void;
|
||||
/** Open settings with the quant the card resolved. */
|
||||
onOpenSettings?: (ggufVariant: string | null) => void;
|
||||
};
|
||||
|
||||
export const ModelInspector = memo(function ModelInspector({
|
||||
|
|
@ -444,6 +446,7 @@ export const ModelInspector = memo(function ModelInspector({
|
|||
onTrain,
|
||||
onInventoryChange,
|
||||
onSearchHub,
|
||||
onOpenSettings,
|
||||
} = actions;
|
||||
const deviceType = usePlatformStore((s) => s.deviceType);
|
||||
const chatOnly = usePlatformStore((s) => s.isChatOnly());
|
||||
|
|
@ -714,6 +717,7 @@ export const ModelInspector = memo(function ModelInspector({
|
|||
model.isDownloaded && canTrainModel ? onTrain : undefined
|
||||
}
|
||||
onChange={onInventoryChange}
|
||||
onOpenSettings={onOpenSettings}
|
||||
/>
|
||||
) : (
|
||||
<DownloadSection
|
||||
|
|
|
|||
|
|
@ -263,6 +263,7 @@ export function DownloadedList({
|
|||
compact = false,
|
||||
sort,
|
||||
onInventoryChange,
|
||||
onOpenModelSettings,
|
||||
}: {
|
||||
cachedRows: CachedInventoryRow[];
|
||||
localRows: LocalInventoryRow[];
|
||||
|
|
@ -284,6 +285,7 @@ export function DownloadedList({
|
|||
compact?: boolean;
|
||||
sort: InventorySort;
|
||||
onInventoryChange?: () => void;
|
||||
onOpenModelSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void;
|
||||
}) {
|
||||
// Pinned repos surface first regardless of the active sort; the chosen sort
|
||||
// still orders rows within the pinned and unpinned groups.
|
||||
|
|
@ -383,6 +385,7 @@ export function DownloadedList({
|
|||
compact={compact}
|
||||
onSelect={onSelect}
|
||||
onChange={onInventoryChange}
|
||||
onOpenSettings={onOpenModelSettings}
|
||||
/>
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -573,6 +573,7 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
compact = false,
|
||||
onSelect,
|
||||
onChange,
|
||||
onOpenSettings,
|
||||
}: {
|
||||
row: CachedInventoryRow | LocalInventoryRow;
|
||||
selected: boolean;
|
||||
|
|
@ -585,6 +586,8 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
compact?: boolean;
|
||||
onSelect: (id: string) => void;
|
||||
onChange?: () => void;
|
||||
/** Open this model's settings page. Omitted for datasets. */
|
||||
onOpenSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void;
|
||||
}) {
|
||||
const rowModelId =
|
||||
row.kind === "cache"
|
||||
|
|
@ -732,30 +735,39 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
const rowPinned =
|
||||
cacheDeletableRepoId != null &&
|
||||
pinnedKeys.includes(pinKey(cacheDeletableRepoId));
|
||||
// Settings applies to any downloaded model, not just deletable ones, so the menu
|
||||
// renders when either action applies and each item gates itself. `deletableRepoId`
|
||||
// (not the boolean) keeps the non-null narrowing the delete closures rely on.
|
||||
const settingsAction =
|
||||
!isDataset && onOpenSettings ? { onOpen: () => onOpenSettings(row) } : undefined;
|
||||
const deletableRepoId = canDelete ? cacheDeletableRepoId : null;
|
||||
const deleteAction =
|
||||
canDelete && cacheDeletableRepoId ? (
|
||||
deletableRepoId || settingsAction ? (
|
||||
<ModelRowMenu
|
||||
ariaLabel={`More options for ${cacheDeletableRepoId}`}
|
||||
ariaLabel={`More options for ${deletableRepoId ?? rowModelId}`}
|
||||
buttonClassName="pointer-events-auto hub-modal-pe-guard p-2 opacity-0 transition-opacity group-hover/row:opacity-100 focus-visible:opacity-100 data-[state=open]:opacity-100 [@media(pointer:coarse)]:opacity-100"
|
||||
iconClassName="size-4"
|
||||
settings={settingsAction}
|
||||
pin={
|
||||
isDataset
|
||||
isDataset || !deletableRepoId
|
||||
? undefined
|
||||
: {
|
||||
pinned: rowPinned,
|
||||
pinLabel: "Pin to top",
|
||||
unpinLabel: "Unpin",
|
||||
onToggle: () => togglePinned(cacheDeletableRepoId),
|
||||
onToggle: () => togglePinned(deletableRepoId),
|
||||
}
|
||||
}
|
||||
cachePath={isDataset ? undefined : { repoId: cacheDeletableRepoId }}
|
||||
del={{
|
||||
cachePath={
|
||||
isDataset || !deletableRepoId ? undefined : { repoId: deletableRepoId }
|
||||
}
|
||||
del={deletableRepoId ? {
|
||||
title: isDataset ? "Delete cached dataset?" : "Delete cached model?",
|
||||
description: (
|
||||
<>
|
||||
This will remove{" "}
|
||||
<span className="font-medium text-foreground">
|
||||
{cacheDeletableRepoId}
|
||||
{deletableRepoId}
|
||||
</span>{" "}
|
||||
{isDataset
|
||||
? "and its downloaded files"
|
||||
|
|
@ -766,17 +778,17 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
disk. You can re-download it later.
|
||||
</>
|
||||
),
|
||||
successMessage: `Deleted ${cacheDeletableRepoId}`,
|
||||
successMessage: `Deleted ${deletableRepoId}`,
|
||||
onConfirm: async () => {
|
||||
// Delete only the copy this row shows: cache rows carry the owning
|
||||
// cache path, so pass it through and leave other caches untouched.
|
||||
const rowCachePath =
|
||||
row.kind === "cache" ? (row.cachePath ?? undefined) : undefined;
|
||||
if (isDataset) {
|
||||
await deleteCachedDataset(cacheDeletableRepoId, rowCachePath);
|
||||
await deleteCachedDataset(deletableRepoId, rowCachePath);
|
||||
} else {
|
||||
await deleteCachedModel(
|
||||
cacheDeletableRepoId,
|
||||
deletableRepoId,
|
||||
undefined,
|
||||
undefined,
|
||||
rowCachePath,
|
||||
|
|
@ -787,11 +799,11 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
usePinnedModelsStore.getState();
|
||||
for (const key of pinned) {
|
||||
if (
|
||||
key === pinKey(cacheDeletableRepoId) ||
|
||||
key.startsWith(`${cacheDeletableRepoId}::`)
|
||||
key === pinKey(deletableRepoId) ||
|
||||
key.startsWith(`${deletableRepoId}::`)
|
||||
) {
|
||||
toggle(
|
||||
cacheDeletableRepoId,
|
||||
deletableRepoId,
|
||||
key.includes("::")
|
||||
? key.slice(key.indexOf("::") + 2)
|
||||
: undefined,
|
||||
|
|
@ -801,7 +813,7 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
}
|
||||
},
|
||||
onDeleted: onChange,
|
||||
}}
|
||||
} : undefined}
|
||||
/>
|
||||
) : null;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import {
|
|||
import type {
|
||||
CachedInventoryRow,
|
||||
DiscoverRow,
|
||||
InventoryRow,
|
||||
LocalInventoryRow,
|
||||
ModelsTab,
|
||||
} from "../types";
|
||||
|
|
@ -70,6 +71,7 @@ export interface ModelsCatalogHandlers {
|
|||
onRetry: () => void;
|
||||
onInventoryChange?: () => void;
|
||||
onSwitchDevice?: () => void;
|
||||
onOpenModelSettings?: (row: InventoryRow) => void;
|
||||
}
|
||||
|
||||
function assignRef<T>(ref: RefObject<T | null>, value: T | null) {
|
||||
|
|
@ -128,6 +130,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({
|
|||
onRetry,
|
||||
onInventoryChange,
|
||||
onSwitchDevice,
|
||||
onOpenModelSettings,
|
||||
} = handlers;
|
||||
const [scrolled, setScrolled] = useState(false);
|
||||
const [streamingActive, setStreamingActive] = useState(false);
|
||||
|
|
@ -483,6 +486,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({
|
|||
columns={discoverView === "two" ? 2 : 1}
|
||||
sort={inventorySort}
|
||||
onInventoryChange={onInventoryChange}
|
||||
onOpenModelSettings={onOpenModelSettings}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
|
|
|
|||
|
|
@ -3,33 +3,29 @@
|
|||
|
||||
import { usePlatformStore } from "@/config/env";
|
||||
import {
|
||||
isChannelEntryFresh,
|
||||
useHubFeedStore,
|
||||
} from "./stores/hub-feed-store";
|
||||
import {
|
||||
applyActiveModelStatusToStore,
|
||||
getInferenceStatus,
|
||||
isExternalModelId,
|
||||
listGgufVariants,
|
||||
resolveInferenceCheckpointId,
|
||||
useChatModelRuntime,
|
||||
useChatRuntimeStore,
|
||||
} from "@/features/chat";
|
||||
import { useHubInventory } from "./inventory";
|
||||
import type {
|
||||
HfModelSearchChannel,
|
||||
HfSortDirection,
|
||||
HfSortKey,
|
||||
} from "./hooks/use-hub-model-search";
|
||||
import { useOnlineStatus } from "@/features/hub";
|
||||
import { useHubInfiniteScroll } from "@/features/hub";
|
||||
import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity";
|
||||
import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store";
|
||||
import {
|
||||
type ModelPickTarget,
|
||||
type PerModelConfig,
|
||||
applyModelLoadConfigToRuntime,
|
||||
applyPerModelConfigToRuntime,
|
||||
currentRuntimePerModelConfig,
|
||||
hfModelFitsDevice,
|
||||
resolveInitialConfig,
|
||||
useActiveModelConfig,
|
||||
} from "@/features/model-picker";
|
||||
import { useDebouncedValue } from "@/hooks/use-debounced-value";
|
||||
import { useGpuInfo, useInferenceGpuInfo } from "@/hooks/use-gpu-info";
|
||||
import { toast } from "@/lib/toast";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useNavigate, useSearch } from "@tanstack/react-router";
|
||||
import {
|
||||
|
|
@ -43,6 +39,7 @@ import {
|
|||
import { ExternalLinkConfirmDialog } from "./catalog/external-link-confirm-dialog";
|
||||
import { HubDetailView } from "./catalog/hub-detail-view";
|
||||
import { HubFeed } from "./catalog/hub-feed";
|
||||
import { HubModelSettingsView } from "./catalog/hub-model-settings-view";
|
||||
import { HubTopBar } from "./catalog/hub-top-bar";
|
||||
import {
|
||||
ModelsCatalog,
|
||||
|
|
@ -66,8 +63,18 @@ import { useDiscoverSearch } from "./hooks/use-discover-search";
|
|||
import { useFeedWriteBack } from "./hooks/use-feed-write-back";
|
||||
import { useHiddenEmbeddingModelIds } from "./hooks/use-hidden-embedding-models";
|
||||
import { useHubFeed } from "./hooks/use-hub-feed";
|
||||
import type {
|
||||
HfModelSearchChannel,
|
||||
HfSortDirection,
|
||||
HfSortKey,
|
||||
} from "./hooks/use-hub-model-search";
|
||||
import { useHubModelVram } from "./hooks/use-hub-model-vram";
|
||||
import { useModelsSelection } from "./hooks/use-models-selection";
|
||||
import { useHubInventory } from "./inventory";
|
||||
import { LOCAL_MODEL_SOURCE } from "./inventory/constants";
|
||||
import { settingsGgufVariantForRow } from "./inventory/settings-identity";
|
||||
import { adoptResidentModelStatus } from "./lib/adopt-inference-status";
|
||||
import { subscribeResidentStatusRefresh } from "./lib/resident-status-refresh";
|
||||
import {
|
||||
CHANNEL_TO_SECTION,
|
||||
type ChannelId,
|
||||
|
|
@ -82,6 +89,11 @@ import {
|
|||
isHiddenModelId,
|
||||
} from "./lib/hidden-models";
|
||||
import { inventoryRowMatches, tokenizeQuery } from "./lib/inventory-search";
|
||||
import {
|
||||
ggufVariantsMatch,
|
||||
modelIdsMatch,
|
||||
residentModelIdMatches,
|
||||
} from "./lib/model-identity";
|
||||
import {
|
||||
type ModelTypeFilter,
|
||||
matchesModelType,
|
||||
|
|
@ -95,6 +107,8 @@ import {
|
|||
matchesCapability,
|
||||
matchesFormat,
|
||||
} from "./lib/view-models";
|
||||
import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store";
|
||||
import { isChannelEntryFresh, useHubFeedStore } from "./stores/hub-feed-store";
|
||||
import type {
|
||||
CachedInventoryRow,
|
||||
CapabilityFilter,
|
||||
|
|
@ -104,8 +118,29 @@ import type {
|
|||
ModelsTab,
|
||||
ResourceTypeFilter,
|
||||
SelectedModelView,
|
||||
SelectedResourceRef,
|
||||
} from "./types";
|
||||
|
||||
// What per-model settings are keyed by, which is not always what the loader is
|
||||
// handed: a repo cached outside the active HF cache loads by snapshot path, while
|
||||
// the chat picker and the auto-switch index key it by repo id, so saving under the
|
||||
// path strands the settings. Local rows are keyed by load id in both places.
|
||||
// The row decides that, not the view it is being shown in: the same cached repo
|
||||
// is reachable from Discover, where the kind is "discover" while the resource is
|
||||
// still the cache row with its snapshot-path run id. Keying that off the kind
|
||||
// stranded the settings for exactly the case this exists to handle. `hub_cache`
|
||||
// is set only for a cached repo; a local row in the active HF cache is `hf_cache`
|
||||
// and stays on its run id, as the Downloaded row keys it.
|
||||
function modelConfigIdentity(
|
||||
kind: SelectedModelView["kind"],
|
||||
resource: SelectedResourceRef,
|
||||
): string {
|
||||
if (kind !== "cache" && resource.source !== "hub_cache") {
|
||||
return resource.runId;
|
||||
}
|
||||
return resource.repoId ?? resource.runId;
|
||||
}
|
||||
|
||||
const MODELS_TAB_STORAGE_KEY = "unsloth.hub.modelsTab";
|
||||
const ALL_MODELS_VIEW_STORAGE_KEY = "unsloth.hub.allModelsView";
|
||||
const INVENTORY_SORT_STORAGE_KEY = "unsloth.hub.inventorySort";
|
||||
|
|
@ -348,33 +383,81 @@ export function ModelsPage() {
|
|||
const activeCheckpoint =
|
||||
checkpoint && !isExternalModelId(checkpoint) ? checkpoint : null;
|
||||
const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant);
|
||||
const activeGgufContextLength = useChatRuntimeStore(
|
||||
(s) => s.ggufContextLength,
|
||||
);
|
||||
// Live settings of the loaded model, so its settings page shows what it is
|
||||
// running with rather than the last saved draft.
|
||||
const { config: activeModelConfig } = useActiveModelConfig();
|
||||
// Shared with the chat model selector: list only models sized for this device.
|
||||
const fitOnDeviceOnly = useChatRuntimeStore((s) => s.fitOnDeviceOnly);
|
||||
const setFitOnDeviceOnly = useChatRuntimeStore((s) => s.setFitOnDeviceOnly);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void getInferenceStatus()
|
||||
// Drops a response that lands after a newer read started, or after unmount.
|
||||
const residentStatusSeq = useRef(0);
|
||||
// Returns the read so a caller that needs the answer before it decides
|
||||
// anything can wait for it; fire-and-forget callers just drop it.
|
||||
const refreshResidentModelStatus = useCallback((): Promise<void> => {
|
||||
const seq = ++residentStatusSeq.current;
|
||||
return getInferenceStatus()
|
||||
.then((status) => {
|
||||
if (cancelled || !status.active_model) return;
|
||||
if (seq !== residentStatusSeq.current) return;
|
||||
const store = useChatRuntimeStore.getState();
|
||||
if (
|
||||
!isExternalModelId(store.params.checkpoint) &&
|
||||
(!modelIdsMatch(store.params.checkpoint, status.active_model) ||
|
||||
!ggufVariantsMatch(
|
||||
store.activeGgufVariant,
|
||||
status.gguf_variant ?? null,
|
||||
))
|
||||
) {
|
||||
store.setCheckpoint(status.active_model, status.gguf_variant ?? null);
|
||||
}
|
||||
adoptResidentModelStatus(
|
||||
{
|
||||
// The loadable identifier, as every other status reader records it: a
|
||||
// GGUF from a non-active HF cache or straight off disk loads by path,
|
||||
// while active_model is the clean public id (an HF snapshot's repo id,
|
||||
// any other file's filename stem). Two files that share a stem collapse
|
||||
// onto one id, so storing that would make the catalog row for one of
|
||||
// them look loaded.
|
||||
checkpointId: resolveInferenceCheckpointId(status),
|
||||
ggufVariant: status.gguf_variant ?? null,
|
||||
},
|
||||
{
|
||||
checkpoint: store.params.checkpoint,
|
||||
checkpointIsExternal: isExternalModelId(store.params.checkpoint),
|
||||
activeGgufVariant: store.activeGgufVariant,
|
||||
modelLoading: store.modelLoading,
|
||||
},
|
||||
{
|
||||
setCheckpoint: (checkpointId, ggufVariant) => {
|
||||
store.setCheckpoint(checkpointId, ggufVariant);
|
||||
},
|
||||
clearCheckpoint: () => {
|
||||
store.clearCheckpoint();
|
||||
},
|
||||
// Landing here is the one entry point that has applied no status yet,
|
||||
// so the settings page would read this model's live config off a store
|
||||
// still holding defaults. Same call the chat runtime's refresh makes.
|
||||
applyStatus: (previous) => {
|
||||
applyActiveModelStatusToStore(status, {
|
||||
previousCheckpoint: previous.checkpoint ?? undefined,
|
||||
previousGgufVariant: previous.ggufVariant,
|
||||
});
|
||||
},
|
||||
},
|
||||
);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Mount, then again whenever this tab could have missed an API-driven switch.
|
||||
// adoptResidentModelStatus is what makes re-reading safe: it stands down for an
|
||||
// external selection and for a load this tab started, so a refresh never fights
|
||||
// the model the user is switching to.
|
||||
useEffect(() => {
|
||||
void refreshResidentModelStatus();
|
||||
const unsubscribe = subscribeResidentStatusRefresh(
|
||||
refreshResidentModelStatus,
|
||||
);
|
||||
return () => {
|
||||
// A response still in flight adopts nothing once the Hub is gone.
|
||||
residentStatusSeq.current += 1;
|
||||
unsubscribe();
|
||||
};
|
||||
}, [refreshResidentModelStatus]);
|
||||
|
||||
const { tab, setTab: setModelsTab } = useModelsTabState();
|
||||
const [query, setQuery] = useState("");
|
||||
const [sortBy, setSortBy] = useState<HfSortKey>(
|
||||
|
|
@ -660,11 +743,7 @@ export function ModelsPage() {
|
|||
const visibleResults =
|
||||
results.length === 0 &&
|
||||
liveListChannel &&
|
||||
isChannelEntryFresh(
|
||||
cachedListEntry,
|
||||
liveListChannel.id,
|
||||
tokenFingerprint,
|
||||
)
|
||||
isChannelEntryFresh(cachedListEntry, liveListChannel.id, tokenFingerprint)
|
||||
? (cachedListEntry?.results ?? results)
|
||||
: results;
|
||||
|
||||
|
|
@ -1187,7 +1266,10 @@ export function ModelsPage() {
|
|||
(opts: ModelLoadOptions, isDownloaded: boolean) => {
|
||||
if (!selectedModel) return;
|
||||
const runId = selectedModel.resource.runId;
|
||||
const resolvedConfig = resolveInitialConfig(runId, opts.ggufVariant);
|
||||
const resolvedConfig = resolveInitialConfig(
|
||||
modelConfigIdentity(selectedModel.kind, selectedModel.resource),
|
||||
opts.ggufVariant,
|
||||
);
|
||||
const rememberedConfig = resolvedConfig.remembered
|
||||
? resolvedConfig.config
|
||||
: null;
|
||||
|
|
@ -1221,6 +1303,177 @@ export function ModelsPage() {
|
|||
runSelectedModel(opts, selectedModel?.isDownloaded ?? true),
|
||||
[runSelectedModel, selectedModel],
|
||||
);
|
||||
|
||||
// Full-page per-model settings, opened from a downloaded row's menu. Local state
|
||||
// rather than a URL param: the page is a transient editor over the catalog, and a
|
||||
// deep link would need the row re-resolved against an inventory that may not have
|
||||
// loaded.
|
||||
const [settingsTarget, setSettingsTarget] = useState<ModelPickTarget | null>(
|
||||
null,
|
||||
);
|
||||
// Opening a model's settings is where a stale read costs something: the editor
|
||||
// is seeded once, from saved/default values when the store says this model is
|
||||
// not the resident one, and Apply reloads with them. Reading status here covers
|
||||
// a switch that landed while this window kept focus the whole time.
|
||||
//
|
||||
// It cannot cover the resolution of the target itself, though: a target has to
|
||||
// exist before this runs. openModelSettings therefore does its own read; this
|
||||
// one is for the handlers that hand the target over ready-made.
|
||||
useEffect(() => {
|
||||
if (settingsTarget) void refreshResidentModelStatus();
|
||||
}, [settingsTarget, refreshResidentModelStatus]);
|
||||
// Bumped per open so a slow variant lookup for an abandoned row cannot land
|
||||
// on top of the row actually chosen.
|
||||
const settingsOpenSeq = useRef(0);
|
||||
const openModelSettings = useCallback(
|
||||
async (row: CachedInventoryRow | LocalInventoryRow) => {
|
||||
const openSeq = ++settingsOpenSeq.current;
|
||||
// loadId is what the loader accepts; repoId is only a display/API alias.
|
||||
const id = row.loadId;
|
||||
// Every name this row answers to. Residency is judged against the store
|
||||
// AFTER the variant lookup settles, not here, because that lookup and the
|
||||
// status refresh this same click starts are in flight together.
|
||||
const rowAliases =
|
||||
row.kind === "local"
|
||||
? [id, row.repoId, row.path]
|
||||
: [id, row.repoId, row.cachePath];
|
||||
// Cached repo rows never carry a quant (cache_inventory.py emits one row per
|
||||
// repo with format_variant null). Opening with a null variant keys the config
|
||||
// to `repo::` while the loader reads `repo::Q4_K_M`, so it never applies and
|
||||
// the server mirror is wrong too. Resolve it as the on-device card does.
|
||||
let ggufVariant = settingsGgufVariantForRow(row);
|
||||
if (!ggufVariant && row.isGguf && row.capabilities.requiresVariant) {
|
||||
// A local row only carries a repo id when it sits in the HF cache, so a
|
||||
// plain folder of quants (the models dir, a custom folder, LM Studio) has
|
||||
// none while still being marked as needing one. The listing takes a path
|
||||
// in the same position and scans it, which is exactly what the on-device
|
||||
// card already does for these rows, so without the fallback this menu
|
||||
// entry could only ever reach the "couldn't determine which quant" toast.
|
||||
const repoId =
|
||||
row.kind === "cache" ? row.repoId : (row.repoId ?? row.path ?? null);
|
||||
if (repoId) {
|
||||
try {
|
||||
const [res] = await Promise.all([
|
||||
listGgufVariants(repoId, hfApiToken(hfToken), {
|
||||
preferLocalCache: true,
|
||||
localPath:
|
||||
row.kind === "local" ? row.path : (row.cachePath ?? null),
|
||||
}),
|
||||
// Read status as part of this click. The effect above cannot help
|
||||
// here, because it does not run until the target it watches exists,
|
||||
// and the Hub has no polling timer: it re-reads on focus and
|
||||
// visibility only. So a window that has kept focus since the last
|
||||
// read holds a checkpoint from before any API-driven switch, for
|
||||
// however long the user has been sitting on the page. Alongside the
|
||||
// variant lookup rather than before it, since that one usually hits
|
||||
// the network while this is a loopback call.
|
||||
refreshResidentModelStatus(),
|
||||
]);
|
||||
const downloaded = res.variants.filter((v) => v.downloaded);
|
||||
// Read residency after the awaits, never from the values closed over
|
||||
// above: the status read that just settled is what knows which model
|
||||
// is resident now, and the loaded-quant branch below would otherwise
|
||||
// silently pick the quant of whichever model it displaced.
|
||||
const settled = useChatRuntimeStore.getState();
|
||||
const settledCheckpoint =
|
||||
settled.params.checkpoint &&
|
||||
!isExternalModelId(settled.params.checkpoint)
|
||||
? settled.params.checkpoint
|
||||
: null;
|
||||
const settledIsActive = rowAliases.some((alias) =>
|
||||
modelIdsMatch(alias, settledCheckpoint),
|
||||
);
|
||||
ggufVariant =
|
||||
// Loaded quant, then the repo default, then whatever is on disk,
|
||||
// mirroring LocalOnDeviceCard's selectedQuant. Only for the loaded
|
||||
// row: Q4_K_M exists in most repos, so an unguarded match would
|
||||
// target the wrong quant of the wrong model.
|
||||
(settledIsActive
|
||||
? downloaded.find((v) =>
|
||||
ggufVariantsMatch(v.quant, settled.activeGgufVariant),
|
||||
)?.quant
|
||||
: undefined) ??
|
||||
downloaded.find((v) =>
|
||||
ggufVariantsMatch(v.quant, res.default_variant),
|
||||
)?.quant ??
|
||||
downloaded[0]?.quant ??
|
||||
null;
|
||||
} catch {
|
||||
ggufVariant = null;
|
||||
}
|
||||
}
|
||||
if (!ggufVariant) {
|
||||
// A model that needs a quant cannot be configured without one: the
|
||||
// picker matches variants exactly and would never find the config,
|
||||
// while the API's bare-key fallback would apply it.
|
||||
toast.error("Couldn't determine which quant to configure.", {
|
||||
description:
|
||||
"Settings for this model are per quant. Check the connection or the model's cache, then try again.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
// The variant lookup is async, so without this a second row opened while it
|
||||
// was pending would be overwritten by whichever call finished last.
|
||||
if (settingsOpenSeq.current !== openSeq) {
|
||||
return;
|
||||
}
|
||||
// A repo in a previous cache loads by snapshot path, so `id` ends in the
|
||||
// revision hash; name the row by what the user calls it instead.
|
||||
const configId = row.kind === "cache" ? row.repoId : id;
|
||||
const leaf = configId.split(/[\\/]/).filter(Boolean).pop() ?? configId;
|
||||
setSettingsTarget({
|
||||
id,
|
||||
configId,
|
||||
displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf,
|
||||
ggufVariant,
|
||||
isGguf: row.isGguf,
|
||||
apiLoadable:
|
||||
row.isGguf &&
|
||||
(row.kind !== "local" || row.source !== LOCAL_MODEL_SOURCE.OLLAMA),
|
||||
meta: {
|
||||
source: "local",
|
||||
isLora: row.modelFormat === "adapter",
|
||||
ggufVariant: ggufVariant ?? undefined,
|
||||
isGguf: row.isGguf,
|
||||
// A partial download opens settings too, but claiming complete would skip
|
||||
// the loader's download-progress reporting.
|
||||
isDownloaded: !row.partial,
|
||||
// Not on inventory rows; ModelConfigPage reads the GGUF header itself.
|
||||
contextLength: null,
|
||||
},
|
||||
});
|
||||
},
|
||||
[activeCheckpoint, activeGgufVariant, hfToken, refreshResidentModelStatus],
|
||||
);
|
||||
// Applying loads the model with exactly these settings. ModelConfigPage has
|
||||
// already persisted them locally and, when "remember" is on, to the server, so
|
||||
// an API request for this model gets the same load.
|
||||
const runSettingsTarget = useCallback(
|
||||
(config: PerModelConfig) => {
|
||||
const target = settingsTarget;
|
||||
if (!target) return;
|
||||
const previousConfig = currentRuntimePerModelConfig({
|
||||
includeMaxSeqLength: true,
|
||||
});
|
||||
applyPerModelConfigToRuntime(config);
|
||||
setSettingsTarget(null);
|
||||
void selectModel({
|
||||
id: target.id,
|
||||
source: "local",
|
||||
ggufVariant: target.ggufVariant ?? undefined,
|
||||
isGguf: target.isGguf,
|
||||
// A partial row opens settings too; claiming complete would skip the
|
||||
// loader's download-progress reporting.
|
||||
isDownloaded: target.meta.isDownloaded,
|
||||
isLora: target.meta.isLora,
|
||||
keepSpeculative: true,
|
||||
forceReload: true,
|
||||
previousConfig,
|
||||
}).catch(() => undefined);
|
||||
},
|
||||
[selectModel, settingsTarget],
|
||||
);
|
||||
const handleLoadLocal = useCallback(
|
||||
(opts: ModelLoadOptions = {}) => runSelectedModel(opts, true),
|
||||
[runSelectedModel],
|
||||
|
|
@ -1228,6 +1481,100 @@ export function ModelsPage() {
|
|||
const handleTrain = useCallback(() => {
|
||||
// Hub → train integration ships in a later PR.
|
||||
}, []);
|
||||
// Opened from the detail view's on-device card, which passes in the quant it
|
||||
// resolved rather than making this re-derive it.
|
||||
const openSelectedModelSettings = useCallback(
|
||||
async (ggufVariant: string | null, quantIsUserPicked = false) => {
|
||||
if (!selectedModel) return;
|
||||
// Share the sequence with openModelSettings: a pending variant lookup for
|
||||
// another row must not land on top of this one.
|
||||
const openSeq = ++settingsOpenSeq.current;
|
||||
let variant = ggufVariant;
|
||||
// The card derived this quant from the store's active variant, and nothing
|
||||
// re-reads status while this window keeps focus, so an API-driven switch
|
||||
// since the last focus event leaves it naming the model that switch
|
||||
// displaced. A quant the user chose in the card is theirs and stands; a
|
||||
// derived one defers to whatever a fresh read says is actually loaded.
|
||||
if (!quantIsUserPicked) {
|
||||
await refreshResidentModelStatus();
|
||||
if (settingsOpenSeq.current !== openSeq) return;
|
||||
const settled = useChatRuntimeStore.getState();
|
||||
const settledCheckpoint =
|
||||
settled.params.checkpoint &&
|
||||
!isExternalModelId(settled.params.checkpoint)
|
||||
? settled.params.checkpoint
|
||||
: null;
|
||||
// Every name this model answers to, as the row menu path matches them.
|
||||
const aliases = [
|
||||
selectedModel.resource.runId,
|
||||
selectedModel.resource.repoId,
|
||||
selectedModel.resource.localPath,
|
||||
];
|
||||
if (
|
||||
settled.activeGgufVariant &&
|
||||
aliases.some((alias) => modelIdsMatch(alias, settledCheckpoint))
|
||||
) {
|
||||
variant = settled.activeGgufVariant;
|
||||
}
|
||||
}
|
||||
// The card passes null while its variant lookup is pending or after it failed,
|
||||
// so this needs the same guard openModelSettings applies: a model that needs a
|
||||
// quant cannot be configured without one.
|
||||
if (!variant && selectedModel.isGguf && selectedModel.requiresVariant) {
|
||||
toast.error("Couldn't determine which quant to configure.", {
|
||||
description:
|
||||
"Settings for this model are per quant. Check the connection or the model's cache, then try again.",
|
||||
});
|
||||
return;
|
||||
}
|
||||
const id = selectedModel.resource.runId;
|
||||
const configId = modelConfigIdentity(
|
||||
selectedModel.kind,
|
||||
selectedModel.resource,
|
||||
);
|
||||
const leaf = configId.split(/[\\/]/).filter(Boolean).pop() ?? configId;
|
||||
setSettingsTarget({
|
||||
id,
|
||||
configId,
|
||||
displayName: variant ? `${leaf} · ${variant}` : leaf,
|
||||
ggufVariant: variant,
|
||||
isGguf: selectedModel.isGguf,
|
||||
apiLoadable:
|
||||
selectedModel.isGguf &&
|
||||
selectedModel.localSource !== LOCAL_MODEL_SOURCE.OLLAMA,
|
||||
meta: {
|
||||
source: "local",
|
||||
isLora: selectedModel.modelFormat === "adapter",
|
||||
ggufVariant: variant ?? undefined,
|
||||
isGguf: selectedModel.isGguf,
|
||||
isDownloaded: selectedModel.isDownloaded,
|
||||
contextLength: null,
|
||||
},
|
||||
});
|
||||
},
|
||||
[selectedModel, refreshResidentModelStatus],
|
||||
);
|
||||
// Whether the settings page is open on the model that is actually loaded, so it
|
||||
// can show the live launch config. A GGUF loaded from an inactive HF cache or
|
||||
// straight off disk loads by path but is reported by its clean public id, so the
|
||||
// row's path and its settings identity both have to be offered as aliases.
|
||||
// A loose .gguf is one file, so its path already names the quant and its
|
||||
// settings deliberately carry no variant. The loader still derives one from the
|
||||
// filename and /status reports it, so requiring the two to agree would never
|
||||
// hold and the page would withhold the live config from the resident file.
|
||||
const settingsTargetIsStandaloneFile =
|
||||
settingsTarget !== null &&
|
||||
settingsTarget.ggufVariant == null &&
|
||||
settingsTarget.id.toLowerCase().endsWith(".gguf");
|
||||
const settingsTargetIsResident =
|
||||
settingsTarget !== null &&
|
||||
residentModelIdMatches(
|
||||
activeCheckpoint,
|
||||
settingsTarget.id,
|
||||
settingsTarget.configId,
|
||||
) &&
|
||||
(settingsTargetIsStandaloneFile ||
|
||||
ggufVariantsMatch(activeGgufVariant, settingsTarget.ggufVariant));
|
||||
const handleSearchHub = useCallback(
|
||||
(next: string) => {
|
||||
const trimmed = next.trim();
|
||||
|
|
@ -1290,6 +1637,7 @@ export function ModelsPage() {
|
|||
onTrain: handleTrain,
|
||||
onInventoryChange: refreshInventory,
|
||||
onSearchHub: handleSearchHub,
|
||||
onOpenSettings: openSelectedModelSettings,
|
||||
}),
|
||||
[
|
||||
handleLoad,
|
||||
|
|
@ -1299,48 +1647,17 @@ export function ModelsPage() {
|
|||
handleTrain,
|
||||
handleSearchHub,
|
||||
refreshInventory,
|
||||
openSelectedModelSettings,
|
||||
],
|
||||
);
|
||||
|
||||
const catalogState = useMemo<ModelsCatalogState>(
|
||||
() => {
|
||||
const typeFilterActive =
|
||||
!isDatasetMode && inventoryTypeFilter !== "all";
|
||||
return {
|
||||
tab,
|
||||
discoverRows: listRows,
|
||||
cachedRows: filteredCachedRows,
|
||||
localRows: filteredLocalRows,
|
||||
selectedId,
|
||||
isLoading,
|
||||
downloadedReady,
|
||||
inventoryError,
|
||||
inventoryWarning,
|
||||
query,
|
||||
activeCheckpoint,
|
||||
activeGgufVariant,
|
||||
searchError,
|
||||
online,
|
||||
isDataset: isDatasetMode,
|
||||
inventoryTokens,
|
||||
scannedCount,
|
||||
loadingIntentCount: discoverFetchIntent,
|
||||
hasMore,
|
||||
manualFetchAvailable: discoverManualFetchAvailable,
|
||||
hasActiveFilters:
|
||||
!isFeedMode &&
|
||||
(deferredFormatFilter !== "all" ||
|
||||
deferredCapabilityFilter !== "all" ||
|
||||
(tab === "downloaded" && typeFilterActive)),
|
||||
typeFilterActive,
|
||||
};
|
||||
},
|
||||
[
|
||||
const catalogState = useMemo<ModelsCatalogState>(() => {
|
||||
const typeFilterActive = !isDatasetMode && inventoryTypeFilter !== "all";
|
||||
return {
|
||||
tab,
|
||||
isFeedMode,
|
||||
listRows,
|
||||
filteredCachedRows,
|
||||
filteredLocalRows,
|
||||
discoverRows: listRows,
|
||||
cachedRows: filteredCachedRows,
|
||||
localRows: filteredLocalRows,
|
||||
selectedId,
|
||||
isLoading,
|
||||
downloadedReady,
|
||||
|
|
@ -1351,17 +1668,45 @@ export function ModelsPage() {
|
|||
activeGgufVariant,
|
||||
searchError,
|
||||
online,
|
||||
isDatasetMode,
|
||||
isDataset: isDatasetMode,
|
||||
inventoryTokens,
|
||||
scannedCount,
|
||||
discoverFetchIntent,
|
||||
loadingIntentCount: discoverFetchIntent,
|
||||
hasMore,
|
||||
discoverManualFetchAvailable,
|
||||
deferredFormatFilter,
|
||||
deferredCapabilityFilter,
|
||||
inventoryTypeFilter,
|
||||
],
|
||||
);
|
||||
manualFetchAvailable: discoverManualFetchAvailable,
|
||||
hasActiveFilters:
|
||||
!isFeedMode &&
|
||||
(deferredFormatFilter !== "all" ||
|
||||
deferredCapabilityFilter !== "all" ||
|
||||
(tab === "downloaded" && typeFilterActive)),
|
||||
typeFilterActive,
|
||||
};
|
||||
}, [
|
||||
tab,
|
||||
isFeedMode,
|
||||
listRows,
|
||||
filteredCachedRows,
|
||||
filteredLocalRows,
|
||||
selectedId,
|
||||
isLoading,
|
||||
downloadedReady,
|
||||
inventoryError,
|
||||
inventoryWarning,
|
||||
query,
|
||||
activeCheckpoint,
|
||||
activeGgufVariant,
|
||||
searchError,
|
||||
online,
|
||||
isDatasetMode,
|
||||
inventoryTokens,
|
||||
scannedCount,
|
||||
discoverFetchIntent,
|
||||
hasMore,
|
||||
discoverManualFetchAvailable,
|
||||
deferredFormatFilter,
|
||||
deferredCapabilityFilter,
|
||||
inventoryTypeFilter,
|
||||
]);
|
||||
|
||||
const catalogPagination = useMemo<ModelsCatalogPagination>(
|
||||
() => ({
|
||||
|
|
@ -1380,6 +1725,7 @@ export function ModelsPage() {
|
|||
onRetry: handleRetrySearch,
|
||||
onInventoryChange: refreshInventory,
|
||||
onSwitchDevice: handleSwitchDevice,
|
||||
onOpenModelSettings: openModelSettings,
|
||||
}),
|
||||
[
|
||||
handleSelect,
|
||||
|
|
@ -1388,6 +1734,7 @@ export function ModelsPage() {
|
|||
handleRetrySearch,
|
||||
refreshInventory,
|
||||
handleSwitchDevice,
|
||||
openModelSettings,
|
||||
],
|
||||
);
|
||||
|
||||
|
|
@ -1530,6 +1877,9 @@ export function ModelsPage() {
|
|||
|
||||
const detailOpen = urlModel !== null;
|
||||
const splitMode = allModelsView === "split";
|
||||
// The catalog is unreachable under an opaque overlay: the detail view (full-page
|
||||
// layout only, since split renders it alongside) or the settings page.
|
||||
const catalogCovered = (detailOpen && !splitMode) || settingsTarget !== null;
|
||||
|
||||
return (
|
||||
<div className="hub-page flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-hidden bg-background">
|
||||
|
|
@ -1585,9 +1935,13 @@ export function ModelsPage() {
|
|||
splitMode
|
||||
? "flex-1 lg:w-[460px] lg:max-w-[44%] lg:flex-none lg:shrink-0 lg:border-r lg:border-border/60"
|
||||
: "flex-1",
|
||||
detailOpen && !splitMode && "pointer-events-none",
|
||||
// The settings page is a full-bleed opaque overlay in every layout,
|
||||
// so it always takes the catalog out of the tab order. Without this,
|
||||
// tabbing out of the form walks into the virtualized rows behind it.
|
||||
catalogCovered && "pointer-events-none",
|
||||
)}
|
||||
aria-hidden={(detailOpen && !splitMode) || undefined}
|
||||
aria-hidden={catalogCovered || undefined}
|
||||
inert={catalogCovered || undefined}
|
||||
>
|
||||
<ModelsCatalog
|
||||
state={catalogState}
|
||||
|
|
@ -1603,7 +1957,10 @@ export function ModelsPage() {
|
|||
|
||||
{splitMode ? (
|
||||
detailOpen ? (
|
||||
<div className="hub-canvas z-20 flex min-h-0 flex-col max-lg:absolute max-lg:inset-0 lg:relative lg:min-w-0 lg:flex-1">
|
||||
<div
|
||||
className="hub-canvas z-20 flex min-h-0 flex-col max-lg:absolute max-lg:inset-0 lg:relative lg:min-w-0 lg:flex-1"
|
||||
inert={settingsTarget !== null || undefined}
|
||||
>
|
||||
<HubDetailView
|
||||
model={selectedModel}
|
||||
preferredGgufFile={preferredGgufFile}
|
||||
|
|
@ -1625,7 +1982,10 @@ export function ModelsPage() {
|
|||
)
|
||||
) : (
|
||||
detailOpen && (
|
||||
<div className="hub-canvas absolute inset-0 z-20 flex min-h-0 flex-col">
|
||||
<div
|
||||
className="hub-canvas absolute inset-0 z-20 flex min-h-0 flex-col"
|
||||
inert={settingsTarget !== null || undefined}
|
||||
>
|
||||
<HubDetailView
|
||||
model={selectedModel}
|
||||
preferredGgufFile={preferredGgufFile}
|
||||
|
|
@ -1641,6 +2001,23 @@ export function ModelsPage() {
|
|||
</div>
|
||||
)
|
||||
)}
|
||||
|
||||
{/* Above the detail overlay (z-30): opening settings while a model
|
||||
preview is open should show the settings, not stack behind it. */}
|
||||
{settingsTarget && (
|
||||
<div className="hub-canvas absolute inset-0 z-30 flex min-h-0 flex-col">
|
||||
<HubModelSettingsView
|
||||
target={settingsTarget}
|
||||
loadedConfig={settingsTargetIsResident ? activeModelConfig : null}
|
||||
loadedContextLength={
|
||||
settingsTargetIsResident ? activeGgufContextLength : null
|
||||
}
|
||||
onBack={() => setSettingsTarget(null)}
|
||||
onRun={runSettingsTarget}
|
||||
compact={splitMode}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<OnDeviceFoldersDialog
|
||||
|
|
|
|||
|
|
@ -41,8 +41,11 @@ export { looksLikeLocalPath } from "./lib/local-path";
|
|||
export { hubTokenHeader } from "./lib/hub-token-header";
|
||||
export {
|
||||
ggufVariantsMatch,
|
||||
isOllamaLinkPath,
|
||||
normalizeGgufVariantIdentity,
|
||||
normalizeModelIdentity,
|
||||
publicModelId,
|
||||
residentModelIdMatches,
|
||||
} from "./lib/model-identity";
|
||||
export { formatBytes, formatRelativeShort } from "./lib/format";
|
||||
export { ggufVariantDisplayLabel } from "./lib/gguf-variant-sort";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// How an inventory row maps onto the identity its saved settings are keyed by.
|
||||
|
||||
import type { CachedInventoryRow, LocalInventoryRow } from "./types";
|
||||
|
||||
/**
|
||||
* The GGUF variant a settings page should key this row's config by, before any
|
||||
* per-repo quant lookup.
|
||||
*
|
||||
* A standalone `.gguf` has no quant to choose between, but the backend inventory
|
||||
* still labels it from its filename (hub/services/models/common.py sets
|
||||
* `format_variant` only when the scanned path is a single file). Adopting that
|
||||
* label would key its settings to `<path>:Q4_K_M` while the Chat model picker, the
|
||||
* detail view's on-device card and the one-time backfill all use the bare path,
|
||||
* leaving two surfaces editing two different configs for one file.
|
||||
*/
|
||||
export function settingsGgufVariantForRow(
|
||||
row: CachedInventoryRow | LocalInventoryRow,
|
||||
): string | null {
|
||||
if (row.kind === "local" && row.path.toLowerCase().endsWith(".gguf")) {
|
||||
return null;
|
||||
}
|
||||
return row.formatVariant?.trim() || null;
|
||||
}
|
||||
107
studio/frontend/src/features/hub/lib/adopt-inference-status.ts
Normal file
107
studio/frontend/src/features/hub/lib/adopt-inference-status.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Adopting the resident model into the chat runtime store from the Hub.
|
||||
//
|
||||
// Landing straight on /hub (a reload, or a deep link after an OpenAI-compatible
|
||||
// auto-switch loaded something else) is the one entry point where nothing has
|
||||
// applied /api/inference/status yet: useChatModelRuntime has no mount sync and
|
||||
// the chat page is a different route. Pinning only the checkpoint leaves every
|
||||
// other field useActiveModelConfig reads at its default, so the Hub's settings
|
||||
// page passes those defaults on as the resident model's live config and Apply
|
||||
// reloads the model with them. Adoption therefore has to apply the whole status,
|
||||
// exactly as the chat runtime's refresh does.
|
||||
|
||||
import { ggufVariantsMatch, modelIdsMatch } from "./model-identity.ts";
|
||||
|
||||
/** The parts of the chat runtime store adoption has to look at. */
|
||||
export interface ResidentAdoptionState {
|
||||
/** ``params.checkpoint``. */
|
||||
checkpoint: string | null;
|
||||
/** Whether that checkpoint names an external provider's model. */
|
||||
checkpointIsExternal: boolean;
|
||||
/** ``activeGgufVariant``. */
|
||||
activeGgufVariant: string | null;
|
||||
/** ``modelLoading``: a load this tab started still owns the store. */
|
||||
modelLoading: boolean;
|
||||
}
|
||||
|
||||
/** What ``/api/inference/status`` says is resident, already resolved. */
|
||||
export interface ResidentStatusFacts {
|
||||
/** ``resolveInferenceCheckpointId(status)``; null when nothing is loaded. */
|
||||
checkpointId: string | null;
|
||||
/** ``status.gguf_variant``. */
|
||||
ggufVariant: string | null;
|
||||
}
|
||||
|
||||
export interface ResidentAdoptionActions {
|
||||
/** Re-pin ``params.checkpoint`` onto the resident model. */
|
||||
setCheckpoint: (checkpointId: string, ggufVariant: string | null) => void;
|
||||
/**
|
||||
* Drop a local checkpoint the server no longer has.
|
||||
*
|
||||
* Optional so a caller that only wants the pinning half can leave it out.
|
||||
*/
|
||||
clearCheckpoint?: () => void;
|
||||
/**
|
||||
* Apply the rest of the status. Receives the store values from BEFORE
|
||||
* ``setCheckpoint`` ran, which is what applyActiveModelStatusToStore needs to
|
||||
* tell a hydration from steady state.
|
||||
*/
|
||||
applyStatus: (previous: {
|
||||
checkpoint: string | null;
|
||||
ggufVariant: string | null;
|
||||
}) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adopt the resident model reported by ``/api/inference/status``.
|
||||
*
|
||||
* Returns whether anything was adopted. Never loads or unloads a model: it only
|
||||
* mirrors what the server already has.
|
||||
*/
|
||||
export function adoptResidentModelStatus(
|
||||
status: ResidentStatusFacts,
|
||||
state: ResidentAdoptionState,
|
||||
actions: ResidentAdoptionActions,
|
||||
): boolean {
|
||||
const { checkpointId } = status;
|
||||
// An external-provider selection has no local mirror, so stamping the resident
|
||||
// GGUF's capabilities and launch settings onto it would describe a model the
|
||||
// user is not talking to. It also owns the store, so an empty status must not
|
||||
// clear it: clearCheckpoint drops the persisted external pick as well.
|
||||
if (state.checkpointIsExternal) {
|
||||
return false;
|
||||
}
|
||||
// A load this tab started applies its own status when it settles, and the load
|
||||
// dialog owns the params meanwhile. Adopting underneath it would fight both.
|
||||
if (state.modelLoading) {
|
||||
return false;
|
||||
}
|
||||
if (!checkpointId) {
|
||||
// The server has nothing loaded, so neither should we. Unloading from another
|
||||
// tab, from the monitor or over the API leaves this store pinned otherwise,
|
||||
// and the settings page goes on treating that row as resident and seeding the
|
||||
// editor from a launch config nothing is running.
|
||||
if (state.checkpoint) {
|
||||
actions.clearCheckpoint?.();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const previous = {
|
||||
checkpoint: state.checkpoint,
|
||||
ggufVariant: state.activeGgufVariant,
|
||||
};
|
||||
const alreadyPinned =
|
||||
modelIdsMatch(previous.checkpoint, checkpointId) &&
|
||||
ggufVariantsMatch(previous.ggufVariant, status.ggufVariant);
|
||||
if (!alreadyPinned) {
|
||||
actions.setCheckpoint(checkpointId, status.ggufVariant);
|
||||
}
|
||||
// Unconditional, even when the checkpoint already matched: a persisted
|
||||
// checkpoint rehydrates from localStorage on its own, with none of the fields
|
||||
// that say how the model was actually launched.
|
||||
actions.applyStatus(previous);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -63,3 +63,138 @@ export function ggufVariantsMatch(
|
|||
normalizeGgufVariantIdentity(left) === normalizeGgufVariantIdentity(right)
|
||||
);
|
||||
}
|
||||
|
||||
// Mirrors core/inference/model_ids.py _looks_like_path.
|
||||
const PUBLIC_ID_PATH_PREFIX_RE = /^(?:[/\\]|\.{1,2}[\\/]|~)/;
|
||||
const GGUF_SUFFIX_RE = /\.gguf$/i;
|
||||
const BACKSLASHES_RE = /\\/g;
|
||||
const TRAILING_SLASHES_RE = /\/+$/;
|
||||
|
||||
function looksLikeModelPath(identifier: string): boolean {
|
||||
if (GGUF_SUFFIX_RE.test(identifier)) {
|
||||
return true;
|
||||
}
|
||||
if (PUBLIC_ID_PATH_PREFIX_RE.test(identifier)) {
|
||||
return true;
|
||||
}
|
||||
if (identifier.length >= 2 && identifier[1] === ":") {
|
||||
return true;
|
||||
}
|
||||
return identifier.split("/").length - 1 >= 2 || identifier.includes("\\");
|
||||
}
|
||||
|
||||
/** `.../models--org--name/snapshots/<sha>` -> `org/name`, else null. */
|
||||
function hfCacheRepoId(path: string): string | null {
|
||||
const parts = path.replace(BACKSLASHES_RE, "/").split("/");
|
||||
for (let index = 0; index < parts.length; index += 1) {
|
||||
const part = parts[index];
|
||||
if (part.startsWith("models--") && parts[index + 1] === "snapshots") {
|
||||
return part.slice("models--".length).replaceAll("--", "/");
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The clean id the backend reports for a model loaded by path.
|
||||
*
|
||||
* Mirrors ``public_model_id`` in studio/backend/core/inference/model_ids.py, which
|
||||
* is what ``/api/inference/status`` puts in ``active_model``: an HF cache snapshot
|
||||
* becomes its repo id and any other local GGUF becomes its filename stem. Repo ids
|
||||
* and already-clean names come back unchanged.
|
||||
*/
|
||||
export function publicModelId(identifier: string): string {
|
||||
const trimmed = identifier.trim();
|
||||
if (!(trimmed && looksLikeModelPath(trimmed))) {
|
||||
return trimmed;
|
||||
}
|
||||
const repoId = hfCacheRepoId(trimmed);
|
||||
if (repoId) {
|
||||
return repoId;
|
||||
}
|
||||
const slashPath = trimmed
|
||||
.replace(BACKSLASHES_RE, "/")
|
||||
.replace(TRAILING_SLASHES_RE, "");
|
||||
const name = slashPath.slice(slashPath.lastIndexOf("/") + 1);
|
||||
return name.replace(GGUF_SUFFIX_RE, "") || trimmed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the model the backend reports as loaded is one of *candidates*.
|
||||
*
|
||||
* A GGUF loaded from an inactive HF cache is loaded by path, but a caller holding
|
||||
* only the public id would read an exact comparison against the catalog row's path
|
||||
* as "not loaded" and fall back to saved or default values instead of the live
|
||||
* launch config. Candidates are compared literally first, then by the public id.
|
||||
*
|
||||
* That second pass only accepts an identity that can name one model: an HF cache
|
||||
* snapshot collapses onto its repo id, which is globally unique, while every other
|
||||
* path collapses onto a filename or directory stem that two models can share
|
||||
* (`/models/alpha/model.gguf` and `/models/beta/model.gguf` are both "model").
|
||||
* Accepting a stem would mark the wrong row resident, seeding its editor with
|
||||
* another model's live config and saving it under this model's key. Callers with
|
||||
* the loadable identifier (`/status`'s `model_identifier`) pass it as the active
|
||||
* id, and the literal pass answers exactly.
|
||||
*/
|
||||
export function residentModelIdMatches(
|
||||
activeModelId: string | null | undefined,
|
||||
...candidates: (string | null | undefined)[]
|
||||
): boolean {
|
||||
if (candidates.some((candidate) => modelIdsMatch(activeModelId, candidate))) {
|
||||
return true;
|
||||
}
|
||||
const active = activeModelId?.trim();
|
||||
// A path-shaped active id is the raw identifier, which the literal pass covered.
|
||||
if (!active || looksLikeModelPath(active)) {
|
||||
return false;
|
||||
}
|
||||
return candidates.some((candidate) => {
|
||||
const trimmed = candidate?.trim();
|
||||
if (!trimmed) {
|
||||
return false;
|
||||
}
|
||||
const publicId = publicModelId(trimmed);
|
||||
// Unambiguous only when the collapse produced a namespaced repo id.
|
||||
return publicId.includes("/") && modelIdsMatch(active, publicId);
|
||||
});
|
||||
}
|
||||
|
||||
// Ollama's blobs reach the picker through a ".studio_links"/"ollama_links" symlink
|
||||
// directory. core/inference/local_model_resolver.py refuses to index anything under
|
||||
// those (the scanner that creates them runs off the request path), so the API can
|
||||
// never load one and mirroring its settings would advertise a load that cannot happen.
|
||||
const OLLAMA_LINK_SEGMENTS = new Set([".studio_links", "ollama_links"]);
|
||||
|
||||
export function isOllamaLinkPath(modelId: string | null | undefined): boolean {
|
||||
if (!modelId) {
|
||||
return false;
|
||||
}
|
||||
return modelId
|
||||
.replace(BACKSLASHES_RE, "/")
|
||||
.split("/")
|
||||
.some((segment) => OLLAMA_LINK_SEGMENTS.has(segment));
|
||||
}
|
||||
|
||||
// A drag-dropped or file-picked GGUF is the API's second unreachable identity.
|
||||
// /api/inference/status reports model_identifier as null for a lease-backed load
|
||||
// (routes/inference.py withholds the host path), so the checkpoint the browser
|
||||
// keys settings by is the bare file name the backend echoes back. _build_index
|
||||
// keys a standalone GGUF by its on-disk path and by its .gguf-stripped stem, so
|
||||
// that name is never an index key and no auto-switch load can read an override
|
||||
// stored under it. Anything the API can load is keyed by a path or a repo id,
|
||||
// both of which carry a separator.
|
||||
const NATIVE_FILE_LABEL_RE = /^[^/\\]+\.gguf$/i;
|
||||
|
||||
export function isNativeFileLabel(modelId: string | null | undefined): boolean {
|
||||
return modelId != null && NATIVE_FILE_LABEL_RE.test(modelId);
|
||||
}
|
||||
|
||||
// A scanned standalone .gguf, keyed by its on-disk path. Its settings identity
|
||||
// carries no variant: it has no quant to choose between, while the loader and the
|
||||
// inventory both label it from its filename, so adopting that label would key one
|
||||
// file's config two ways. settings-identity.ts applies the same rule to a Hub row.
|
||||
export function isStandaloneGgufPath(
|
||||
modelId: string | null | undefined,
|
||||
): boolean {
|
||||
return modelId != null && modelId.toLowerCase().endsWith(".gguf");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// When the Hub has to re-read /api/inference/status.
|
||||
//
|
||||
// An OpenAI-compatible request can auto-switch the resident model at any moment,
|
||||
// and nothing else on /hub reads status: the chat runtime hook has no mount sync
|
||||
// and the chat page is a different route. A mount-only read therefore leaves
|
||||
// every "loaded" marker -- and, worse, the settings page's live config -- pinned
|
||||
// to whatever was resident when the Hub opened, so the newly loaded model's
|
||||
// editor seeds from saved/default values and Apply reloads it with them.
|
||||
//
|
||||
// A background timer would keep asking a server that has usually not changed, so
|
||||
// re-read only on the moments this tab could have missed a switch instead.
|
||||
|
||||
/** The event targets to listen on; injected so this is testable off a browser. */
|
||||
export interface ResidentStatusRefreshTargets {
|
||||
window: Pick<EventTarget, "addEventListener" | "removeEventListener">;
|
||||
document: Pick<EventTarget, "addEventListener" | "removeEventListener"> & {
|
||||
readonly hidden: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
function browserTargets(): ResidentStatusRefreshTargets {
|
||||
return { window, document };
|
||||
}
|
||||
|
||||
/**
|
||||
* Call ``refresh`` whenever this tab comes back to the foreground: returning from
|
||||
* the terminal or client that made the API call is exactly when what is resident
|
||||
* may have moved. Returns the unsubscribe.
|
||||
*/
|
||||
export function subscribeResidentStatusRefresh(
|
||||
refresh: () => void,
|
||||
targets: ResidentStatusRefreshTargets = browserTargets(),
|
||||
): () => void {
|
||||
const onFocus = () => refresh();
|
||||
// Focus alone misses a tab that was merely backgrounded, and visibility alone
|
||||
// misses a window that never went hidden; a redundant pair of reads is cheaper
|
||||
// than a missed switch.
|
||||
const onVisibility = () => {
|
||||
if (!targets.document.hidden) refresh();
|
||||
};
|
||||
targets.window.addEventListener("focus", onFocus);
|
||||
targets.document.addEventListener("visibilitychange", onVisibility);
|
||||
return () => {
|
||||
targets.window.removeEventListener("focus", onFocus);
|
||||
targets.document.removeEventListener("visibilitychange", onVisibility);
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,174 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// One-time backfill of per-model settings into the server override map.
|
||||
//
|
||||
// Settings used to live only in this browser, so on upgrade the server knows
|
||||
// nothing about models already configured: they still show as remembered while an
|
||||
// API load uses app defaults, the exact bug the server-side map exists to fix.
|
||||
|
||||
import {
|
||||
isNativeFileLabel,
|
||||
isOllamaLinkPath,
|
||||
normalizeGgufVariantIdentity,
|
||||
normalizeModelIdentity,
|
||||
splitQuantSuffix,
|
||||
} from "../model-config/model-identity";
|
||||
import {
|
||||
isDefaultConfig,
|
||||
listPerModelConfigs,
|
||||
} from "../model-config/per-model-config";
|
||||
import type { ApiModelOverride } from "./model-overrides";
|
||||
import {
|
||||
fetchModelOverrides,
|
||||
modelOverrideKey,
|
||||
putModelOverride,
|
||||
toApiOverride,
|
||||
} from "./model-overrides";
|
||||
|
||||
const DONE_FLAG = "unsloth_model_overrides_backfilled_v1";
|
||||
|
||||
function alreadyRan(): boolean {
|
||||
try {
|
||||
return window.localStorage.getItem(DONE_FLAG) === "1";
|
||||
} catch {
|
||||
// Storage denied: treat as done rather than re-running on every mount.
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
function markRan(): void {
|
||||
try {
|
||||
window.localStorage.setItem(DONE_FLAG, "1");
|
||||
} catch {
|
||||
// Nothing to do; the backfill is idempotent anyway.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A server key under the same identity this browser stores.
|
||||
*
|
||||
* `app_settings` has no schema version, so an old install holds keys like
|
||||
* `Unsloth/Repo-GGUF:Q4_K_M` that the backend resolves to the same model as this
|
||||
* browser's folded form; an exact lookup would call it missing and let the backfill
|
||||
* overwrite it. The quant-aware split folds repo ids and leaves POSIX paths alone.
|
||||
*/
|
||||
function normalizedOverrideKey(key: string): string {
|
||||
const split = splitQuantSuffix(key);
|
||||
if (!split) {
|
||||
return modelOverrideKey(normalizeModelIdentity(key));
|
||||
}
|
||||
return modelOverrideKey(
|
||||
normalizeModelIdentity(split[0]),
|
||||
normalizeGgufVariantIdentity(split[1]),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The fields *config* would contribute that the stored entry does not hold.
|
||||
*
|
||||
* A malformed entry (nothing constrains what an older install wrote into
|
||||
* app_settings) counts as holding nothing, so the migration still runs.
|
||||
*/
|
||||
function absentFields(
|
||||
stored: ApiModelOverride,
|
||||
config: Parameters<typeof toApiOverride>[0],
|
||||
): string[] {
|
||||
const fields = Object.keys(toApiOverride(config));
|
||||
if (typeof stored !== "object" || stored === null) {
|
||||
return fields;
|
||||
}
|
||||
return fields.filter((field) => !(field in stored));
|
||||
}
|
||||
|
||||
/**
|
||||
* Push local settings the server does not hold. Never deletes and never overwrites:
|
||||
* a value already there is the newer authority, and losing a setting would be worse
|
||||
* than leaving one unmigrated.
|
||||
*
|
||||
* Field by field, not entry by entry. The override map shipped before this browser
|
||||
* mirror did, storing only llama_extra_args and max_seq_length, so an upgraded
|
||||
* install can hold an entry for a model whose context, KV cache, speculative and GPU
|
||||
* settings live only here. Treating the key as done would skip exactly the settings
|
||||
* this migration exists to carry and then mark it complete.
|
||||
*/
|
||||
export async function backfillModelOverrides(): Promise<void> {
|
||||
if (alreadyRan()) {
|
||||
return;
|
||||
}
|
||||
const local = listPerModelConfigs().filter(
|
||||
// A quant means GGUF, the only thing API auto-switch resolves, so backfilling a
|
||||
// safetensors config would claim behaviour that does not exist. A standalone
|
||||
// .gguf has no quant to select between and is stored with a null variant, so it
|
||||
// needs the extra test or its settings stay browser-only for good. An Ollama
|
||||
// blob is GGUF but reached through a link dir the resolver skips, so it is not
|
||||
// auto-switchable either, and a bare file name is a dropped/picked file's label,
|
||||
// which the resolver never keys.
|
||||
(entry) =>
|
||||
(entry.ggufVariant != null ||
|
||||
entry.modelId.toLowerCase().endsWith(".gguf")) &&
|
||||
!isOllamaLinkPath(entry.modelId) &&
|
||||
!isNativeFileLabel(entry.modelId) &&
|
||||
!isDefaultConfig(entry.config),
|
||||
);
|
||||
if (local.length === 0) {
|
||||
markRan();
|
||||
return;
|
||||
}
|
||||
|
||||
let existing: Awaited<ReturnType<typeof fetchModelOverrides>>;
|
||||
try {
|
||||
existing = await fetchModelOverrides();
|
||||
} catch {
|
||||
// Offline or not authenticated yet. Leave the flag unset so the next start
|
||||
// retries rather than skipping the migration forever.
|
||||
return;
|
||||
}
|
||||
|
||||
const known = new Map<string, ApiModelOverride>();
|
||||
for (const [storedKey, storedEntry] of Object.entries(existing)) {
|
||||
known.set(normalizedOverrideKey(storedKey), storedEntry);
|
||||
}
|
||||
|
||||
let failed = false;
|
||||
for (const entry of local) {
|
||||
// Folded here too: a v2 storage key holds the normalized identity, but the
|
||||
// older `id::variant` keys this browser still reads hold the typed casing.
|
||||
const key = normalizedOverrideKey(
|
||||
modelOverrideKey(entry.modelId, entry.ggufVariant),
|
||||
);
|
||||
// Re-read rather than trusting the snapshot from before the fetch: this write
|
||||
// is queued behind the interactive one and commits last, so a save or forget
|
||||
// during the round trip would be undone by it.
|
||||
const current = listPerModelConfigs().find(
|
||||
(candidate) =>
|
||||
normalizedOverrideKey(
|
||||
modelOverrideKey(candidate.modelId, candidate.ggufVariant),
|
||||
) === key,
|
||||
);
|
||||
if (!current || isDefaultConfig(current.config)) {
|
||||
continue;
|
||||
}
|
||||
const stored = known.get(key);
|
||||
// Nothing this browser could add, so skip the round trip entirely.
|
||||
if (stored && absentFields(stored, current.config).length === 0) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
// Fills the gaps only. `known` is a snapshot from before this loop started, so
|
||||
// a save by another tab during the pass is invisible here; the server reads and
|
||||
// writes together rather than this re-fetching once per model.
|
||||
await putModelOverride(
|
||||
current.modelId,
|
||||
current.ggufVariant,
|
||||
current.config,
|
||||
{ fillAbsentFields: true },
|
||||
);
|
||||
} catch {
|
||||
failed = true;
|
||||
}
|
||||
}
|
||||
if (!failed) {
|
||||
markRan();
|
||||
}
|
||||
}
|
||||
252
studio/frontend/src/features/model-picker/api/model-overrides.ts
Normal file
252
studio/frontend/src/features/model-picker/api/model-overrides.ts
Normal file
|
|
@ -0,0 +1,252 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Server-side mirror of the per-model config.
|
||||
//
|
||||
// The config in ../model-config/per-model-config.ts lives in browser localStorage,
|
||||
// so it only applied to loads the browser made, and an API auto-switch load (no
|
||||
// browser in the loop) came up with none of the user's settings. Mirroring every
|
||||
// save to the backend's override map closes that gap: routes/inference.py reads it
|
||||
// and rebuilds the same LoadRequest the picker would have sent.
|
||||
|
||||
import { authFetch } from "@/features/auth";
|
||||
import { readFastApiError } from "@/lib/format-fastapi-error";
|
||||
import {
|
||||
normalizeGgufVariantIdentity,
|
||||
normalizeModelIdentity,
|
||||
} from "../model-config/model-identity";
|
||||
import type { PerModelConfig } from "../model-config/per-model-config";
|
||||
|
||||
const OVERRIDES_URL = "/api/settings/openai-auto-switch/overrides";
|
||||
|
||||
/** One model's stored launch config, as the backend persists it. */
|
||||
export interface ApiModelOverride {
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
llama_extra_args?: string[];
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
max_seq_length?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
custom_context_length?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
kv_cache_dtype?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
speculative_type?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
spec_draft_n_max?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
n_parallel?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
tensor_parallel?: boolean;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
chat_template_override?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
gpu_memory_mode?: "auto" | "manual";
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
gpu_layers?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
n_cpu_moe?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
gpu_ids?: number[];
|
||||
}
|
||||
|
||||
export type ApiModelOverrides = Record<string, ApiModelOverride>;
|
||||
|
||||
/**
|
||||
* The key one model's config is stored under: the `repo:VARIANT` form an OpenAI
|
||||
* request names a quant by, so two quants of one repo keep separate configs and the
|
||||
* backend matches the requested name directly. Bare id when there is no variant.
|
||||
*/
|
||||
export function modelOverrideKey(
|
||||
modelId: string,
|
||||
ggufVariant?: string | null,
|
||||
): string {
|
||||
return ggufVariant ? `${modelId}:${ggufVariant}` : modelId;
|
||||
}
|
||||
|
||||
export async function fetchModelOverrides(): Promise<ApiModelOverrides> {
|
||||
const res = await authFetch(OVERRIDES_URL);
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
await readFastApiError(res, "Failed to load saved model settings"),
|
||||
);
|
||||
}
|
||||
const body = (await res.json()) as { overrides?: ApiModelOverrides };
|
||||
return body.overrides ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate the UI's per-model config into the backend's schema.
|
||||
*
|
||||
* Only fields the user set are sent: the backend reads an absent field as "app
|
||||
* default", so nulls would pin defaults and stop the model following later global
|
||||
* changes. A `null` config means "no saved settings", which clears the entry.
|
||||
*
|
||||
* Exported so the one-time backfill can ask what this config would contribute and
|
||||
* compare it against the entry already on the server, field by field.
|
||||
*/
|
||||
export function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
|
||||
if (!config) {
|
||||
return {};
|
||||
}
|
||||
const payload: ApiModelOverride = {};
|
||||
if (config.maxSeqLength && config.maxSeqLength > 0) {
|
||||
payload.max_seq_length = config.maxSeqLength;
|
||||
}
|
||||
if (config.customContextLength && config.customContextLength > 0) {
|
||||
payload.custom_context_length = config.customContextLength;
|
||||
}
|
||||
if (config.kvCacheDtype) {
|
||||
payload.kv_cache_dtype = config.kvCacheDtype;
|
||||
}
|
||||
if (config.speculativeType) {
|
||||
payload.speculative_type = config.speculativeType;
|
||||
}
|
||||
if (config.specDraftNMax && config.specDraftNMax > 0) {
|
||||
payload.spec_draft_n_max = config.specDraftNMax;
|
||||
}
|
||||
// Blank follows the server-wide --parallel default, which is the app default here.
|
||||
if (config.nParallel && config.nParallel > 0) {
|
||||
payload.n_parallel = config.nParallel;
|
||||
}
|
||||
if (config.tensorParallel) {
|
||||
payload.tensor_parallel = true;
|
||||
}
|
||||
if (config.chatTemplateOverride?.trim()) {
|
||||
payload.chat_template_override = config.chatTemplateOverride;
|
||||
}
|
||||
// Only "manual" is a real override; "auto" is the follow-the-global default.
|
||||
if (config.gpuMemoryMode === "manual") {
|
||||
payload.gpu_memory_mode = "manual";
|
||||
}
|
||||
// gpuLayers < 0 is Auto, which is also the default.
|
||||
if (typeof config.gpuLayers === "number" && config.gpuLayers >= 0) {
|
||||
payload.gpu_layers = config.gpuLayers;
|
||||
}
|
||||
if (typeof config.nCpuMoe === "number" && config.nCpuMoe > 0) {
|
||||
payload.n_cpu_moe = config.nCpuMoe;
|
||||
}
|
||||
if (config.selectedGpuIds && config.selectedGpuIds.length > 0) {
|
||||
payload.gpu_ids = config.selectedGpuIds;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
// One in-flight write per model, so writes for a model commit in issue order.
|
||||
// Otherwise saving twice quickly, or saving during the one-time backfill, races:
|
||||
// the older response can land last and resurrect the entry the newer one meant to
|
||||
// replace. Different models still overlap.
|
||||
const writesByKey = new Map<string, Promise<void>>();
|
||||
|
||||
export interface PutModelOverrideOptions {
|
||||
/**
|
||||
* Fill in only what is missing: every value already on the server stays as it is.
|
||||
*
|
||||
* The one-time backfill reads the map once and then writes each model in turn, so
|
||||
* another tab saving during that pass would be overwritten by this browser's older
|
||||
* localStorage copy. The server reads and writes under one transaction, which
|
||||
* closes the window without a round trip per model. Field level, so an entry an
|
||||
* older release stored with only its two fields still gains the browser-only ones
|
||||
* without the server losing anything it holds.
|
||||
*/
|
||||
fillAbsentFields?: boolean;
|
||||
/**
|
||||
* Clear the fields this UI mirrors but leave the server's own launch flags alone.
|
||||
*
|
||||
* Dropping a local entry to stay inside the storage budget is not the user asking
|
||||
* to forget that model, so it must not take `llama_extra_args` the settings API
|
||||
* set and the settings page can neither show nor restore. The route still drops
|
||||
* the row outright once nothing is left in it.
|
||||
*/
|
||||
keepLaunchFlags?: boolean;
|
||||
}
|
||||
|
||||
export async function putModelOverride(
|
||||
modelId: string,
|
||||
ggufVariant: string | null | undefined,
|
||||
config: PerModelConfig | null,
|
||||
options?: PutModelOverrideOptions,
|
||||
): Promise<void> {
|
||||
// Keyed by the folded identity, not the literal spelling: the backfill sends a
|
||||
// legacy casing and a UI save the normalized one, and the backend resolves both
|
||||
// to one row, so raw strings would open two queues and race again.
|
||||
const key = modelOverrideKey(
|
||||
normalizeModelIdentity(modelId),
|
||||
normalizeGgufVariantIdentity(ggufVariant),
|
||||
);
|
||||
// Chain on the settled tail: a failed write must not cancel the next one.
|
||||
const previous = writesByKey.get(key) ?? Promise.resolve();
|
||||
const write = previous
|
||||
.catch(() => {})
|
||||
.then(() => sendModelOverride(modelId, ggufVariant, config, options));
|
||||
writesByKey.set(key, write);
|
||||
try {
|
||||
await write;
|
||||
} finally {
|
||||
// Only the last writer clears the slot, so a queue still building keeps order.
|
||||
if (writesByKey.get(key) === write) {
|
||||
writesByKey.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function sendModelOverride(
|
||||
modelId: string,
|
||||
ggufVariant: string | null | undefined,
|
||||
config: PerModelConfig | null,
|
||||
options?: PutModelOverrideOptions,
|
||||
): Promise<void> {
|
||||
const res = await authFetch(OVERRIDES_URL, {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
model_id: modelOverrideKey(modelId, ggufVariant),
|
||||
// Only sent when set, so an older backend that does not know the field is
|
||||
// not handed an unexpected key by every ordinary save.
|
||||
...(options?.fillAbsentFields
|
||||
? // biome-ignore lint/style/useNamingConvention: API schema
|
||||
{ fill_absent_fields: true }
|
||||
: {}),
|
||||
// Say which operation this is: an all-default save carries no fields, which is
|
||||
// shape-identical to "forget this model", and guessing wrong wipes launch flags
|
||||
// the UI cannot show or restore.
|
||||
remove: config === null && !options?.keepLaunchFlags,
|
||||
// Launch flags have no UI control, so the backend preserves them when omitted.
|
||||
// Forgetting means forgetting all of it, so that path sends an explicit [].
|
||||
...(config === null && !options?.keepLaunchFlags
|
||||
? // biome-ignore lint/style/useNamingConvention: API schema
|
||||
{ llama_extra_args: [] }
|
||||
: {}),
|
||||
...toApiOverride(config),
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
await readFastApiError(res, "Failed to save model settings for the API"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror a per-model config save to the backend without blocking the UI.
|
||||
*
|
||||
* Best-effort: the localStorage write is this browser's source of truth and has
|
||||
* already happened, so a failed sync must not fail the save or interrupt a load.
|
||||
* Logged rather than toasted: an API load of this model just falls back to app
|
||||
* defaults until the next successful save.
|
||||
*/
|
||||
export function syncModelOverride(
|
||||
modelId: string,
|
||||
ggufVariant: string | null | undefined,
|
||||
config: PerModelConfig | null,
|
||||
options?: PutModelOverrideOptions,
|
||||
): void {
|
||||
void putModelOverride(modelId, ggufVariant, config, options).catch(
|
||||
(error: unknown) => {
|
||||
console.warn(
|
||||
"Failed to mirror model settings to the server; an API load of this model will use defaults.",
|
||||
error,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -32,6 +32,7 @@ import {
|
|||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { syncModelOverride } from "../api/model-overrides";
|
||||
import {
|
||||
useDefaultChatTemplate,
|
||||
useModelMaxPositionEmbeddings,
|
||||
|
|
@ -54,6 +55,7 @@ import {
|
|||
floorMaxSeqLength,
|
||||
isDefaultConfig,
|
||||
normalizeMaxSeqLength,
|
||||
normalizePerModelConfig,
|
||||
resolveInitialConfig,
|
||||
savePerModelConfig,
|
||||
} from "../model-config/per-model-config";
|
||||
|
|
@ -75,14 +77,16 @@ const SELECT_TRIGGER_CLASS = `grid h-8 min-w-0 grid-cols-[minmax(0,1fr)_auto] it
|
|||
const NUMBER_INPUT_CLASS = `h-8 w-[92px] ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-right text-ui-13 font-medium text-nav-fg outline-none focus-visible:ring-0`;
|
||||
|
||||
const KV_CACHE_DTYPE_DEFAULT = "f16";
|
||||
const SPECULATIVE_TYPE_LABELS: Record<(typeof SPECULATIVE_TYPES)[number], string> =
|
||||
{
|
||||
auto: "Auto",
|
||||
mtp: "MTP",
|
||||
ngram: "Ngram",
|
||||
"mtp+ngram": "MTP+Ngram",
|
||||
off: "Off",
|
||||
};
|
||||
const SPECULATIVE_TYPE_LABELS: Record<
|
||||
(typeof SPECULATIVE_TYPES)[number],
|
||||
string
|
||||
> = {
|
||||
auto: "Auto",
|
||||
mtp: "MTP",
|
||||
ngram: "Ngram",
|
||||
"mtp+ngram": "MTP+Ngram",
|
||||
off: "Off",
|
||||
};
|
||||
|
||||
function hasNonDefaultAdvanced(config: PerModelConfig): boolean {
|
||||
return (
|
||||
|
|
@ -354,8 +358,8 @@ function GpuMemorySettings({
|
|||
info={
|
||||
<>
|
||||
Layers to keep on the GPU (--gpu-layers); the rest run on CPU.
|
||||
Auto lets llama.cpp size the split (and the context) to fit VRAM.
|
||||
At the maximum, the whole model is on the GPU.
|
||||
Auto lets llama.cpp size the split (and the context) to fit
|
||||
VRAM. At the maximum, the whole model is on the GPU.
|
||||
</>
|
||||
}
|
||||
/>
|
||||
|
|
@ -619,6 +623,11 @@ interface ModelConfigPageProps {
|
|||
loadedContextLength?: number | null;
|
||||
initialConfig?: PerModelConfig | null;
|
||||
variant?: "page" | "sidebar";
|
||||
/**
|
||||
* Page variant only: render the built-in "Run settings" title block. A host that
|
||||
* already shows the model name as its page heading turns this off.
|
||||
*/
|
||||
showHeader?: boolean;
|
||||
}
|
||||
|
||||
export function ModelConfigPage({
|
||||
|
|
@ -629,6 +638,7 @@ export function ModelConfigPage({
|
|||
loadedContextLength = null,
|
||||
initialConfig = null,
|
||||
variant = "page",
|
||||
showHeader = true,
|
||||
}: ModelConfigPageProps) {
|
||||
const rememberId = useId();
|
||||
const isActiveModel = loadedConfig != null;
|
||||
|
|
@ -642,8 +652,12 @@ export function ModelConfigPage({
|
|||
const loadedMaxContextLength = useChatRuntimeStore(
|
||||
(s) => s.ggufMaxContextLength,
|
||||
);
|
||||
// What the settings are stored under, which is not always what loads (see
|
||||
// ModelPickTarget.configId). Every read, write and mirror uses it; the probes
|
||||
// keep target.id, since they have to open the model.
|
||||
const configId = target.configId ?? target.id;
|
||||
const resolveInitial = () => {
|
||||
const resolved = resolveInitialConfig(target.id, target.ggufVariant);
|
||||
const resolved = resolveInitialConfig(configId, target.ggufVariant);
|
||||
if (loadedConfig) {
|
||||
return { config: loadedConfig, remembered: resolved.remembered };
|
||||
}
|
||||
|
|
@ -771,8 +785,7 @@ export function ModelConfigPage({
|
|||
),
|
||||
maxContext,
|
||||
);
|
||||
const setContextLength = (v: number) =>
|
||||
update({ customContextLength: v });
|
||||
const setContextLength = (v: number) => update({ customContextLength: v });
|
||||
const baseline = loadedConfig ?? DEFAULT_PER_MODEL_CONFIG;
|
||||
const atBaseline = perModelConfigsEqual(config, baseline);
|
||||
// An explicit customContextLength equal to the native ceiling is still an
|
||||
|
|
@ -900,16 +913,56 @@ export function ModelConfigPage({
|
|||
const effectiveAtBaseline = perModelConfigsEqual(effectiveConfig, baseline);
|
||||
const effectivePersistenceOnly =
|
||||
isActiveModel && effectiveAtBaseline && rememberChanged;
|
||||
const defaultConfig = isDefaultConfig(effectiveRuntimeConfig);
|
||||
// Judge what storage keeps: savePerModelConfig normalizes first, and the runtime's
|
||||
// Speculative Decoding "auto" canonicalizes to null, so judging the raw object
|
||||
// reported saved while the write dropped it, and mirrored an override nothing held.
|
||||
const normalizedRuntimeConfig = normalizePerModelConfig(
|
||||
effectiveRuntimeConfig,
|
||||
);
|
||||
const defaultConfig = isDefaultConfig(normalizedRuntimeConfig);
|
||||
let saveFailed = false;
|
||||
const evicted: { modelId: string; ggufVariant: string | null }[] = [];
|
||||
if (remember) {
|
||||
saveFailed = !savePerModelConfig(
|
||||
target.id,
|
||||
configId,
|
||||
target.ggufVariant,
|
||||
effectiveRuntimeConfig,
|
||||
normalizedRuntimeConfig,
|
||||
evicted,
|
||||
);
|
||||
} else {
|
||||
saveFailed = !deletePerModelConfig(target.id, target.ggufVariant);
|
||||
saveFailed = !deletePerModelConfig(configId, target.ggufVariant);
|
||||
}
|
||||
// Mirror to the server so an API request that loads this model gets these exact
|
||||
// settings, not app defaults. Best-effort and non-blocking: the localStorage
|
||||
// write above already governs this browser, and forgetting clears both.
|
||||
//
|
||||
// Skipped when the local write failed (quota, a future-schema entry), or the
|
||||
// two would permanently disagree with no way to tell which the next load used.
|
||||
// Gated on auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and
|
||||
// skips Ollama, so mirroring either would advertise a load that cannot happen.
|
||||
// A native-path lease is the same case: /status withholds model_identifier for a
|
||||
// dropped or file-picked GGUF, so this id is only the file's display name, which
|
||||
// the resolver never keys, and reopening the file needs a lease the API cannot
|
||||
// mint. The token, not the name, decides: the fallback label carries no suffix.
|
||||
if (
|
||||
!saveFailed &&
|
||||
(target.apiLoadable ?? target.isGguf) &&
|
||||
!nativePathToken
|
||||
) {
|
||||
syncModelOverride(
|
||||
configId,
|
||||
target.ggufVariant,
|
||||
remember ? normalizedRuntimeConfig : null,
|
||||
);
|
||||
}
|
||||
// Saving can push the local map over budget and drop other models, whose server
|
||||
// entries would keep being applied with nothing in the UI able to forget them.
|
||||
// Not a Forget though: the user never asked to drop these, so only the mirrored
|
||||
// fields go and launch flags set through the API stay.
|
||||
for (const dropped of evicted) {
|
||||
syncModelOverride(dropped.modelId, dropped.ggufVariant, null, {
|
||||
keepLaunchFlags: true,
|
||||
});
|
||||
}
|
||||
if (effectivePersistenceOnly) {
|
||||
if (saveFailed) {
|
||||
|
|
@ -939,7 +992,7 @@ export function ModelConfigPage({
|
|||
|
||||
return (
|
||||
<div className="flex flex-col">
|
||||
{variant === "page" && (
|
||||
{variant === "page" && showHeader && (
|
||||
<div className="flex items-center gap-2.5 pb-4">
|
||||
{onBack && (
|
||||
<button
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ import {
|
|||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { isOllamaLinkPath } from "../model-config/model-identity";
|
||||
import {
|
||||
type PerModelConfig,
|
||||
resolveInitialConfig,
|
||||
|
|
@ -480,11 +481,16 @@ function ModelSelectorContent({
|
|||
const visibleConfigTarget = open ? configTarget : null;
|
||||
const openConfigPage = (id: string, meta: ModelSelectorChangeMeta) => {
|
||||
const leaf = id.includes("/") ? id.slice(id.lastIndexOf("/") + 1) : id;
|
||||
const isGguf = meta.isGguf ?? Boolean(meta.ggufVariant);
|
||||
setConfigTarget({
|
||||
id,
|
||||
displayName: meta.ggufVariant ? `${leaf} · ${meta.ggufVariant}` : leaf,
|
||||
ggufVariant: meta.ggufVariant ?? null,
|
||||
isGguf: meta.isGguf ?? Boolean(meta.ggufVariant),
|
||||
isGguf,
|
||||
// Ollama's models list here as custom-folder GGUFs under a link dir the
|
||||
// auto-switch resolver skips, so mirroring their settings to the server
|
||||
// would advertise a load the API can never make.
|
||||
apiLoadable: isGguf && !isOllamaLinkPath(id),
|
||||
meta,
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import {
|
|||
MoreVerticalIcon,
|
||||
PinIcon,
|
||||
PinOffIcon,
|
||||
Settings02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { RefreshCw } from "lucide-react";
|
||||
|
|
@ -73,11 +74,17 @@ interface ModelRowMenuCachePath {
|
|||
variant?: string;
|
||||
}
|
||||
|
||||
/** The model's settings page: load config plus what the API will apply. */
|
||||
interface ModelRowMenuSettings {
|
||||
onOpen: () => void;
|
||||
}
|
||||
|
||||
export function ModelRowMenu({
|
||||
ariaLabel,
|
||||
buttonClassName,
|
||||
iconClassName,
|
||||
cachePath,
|
||||
settings,
|
||||
pin,
|
||||
update,
|
||||
del,
|
||||
|
|
@ -87,6 +94,7 @@ export function ModelRowMenu({
|
|||
iconClassName?: string;
|
||||
/** Enables "Reveal in Finder" for cached repos. */
|
||||
cachePath?: ModelRowMenuCachePath;
|
||||
settings?: ModelRowMenuSettings;
|
||||
pin?: ModelRowMenuPin;
|
||||
update?: ModelRowMenuUpdate;
|
||||
del?: ModelRowMenuDelete;
|
||||
|
|
@ -167,7 +175,7 @@ export function ModelRowMenu({
|
|||
});
|
||||
}, [cachePathRepoId, cachePathVariant]);
|
||||
|
||||
if (!pin && !update && !del && !cachePath) return null;
|
||||
if (!pin && !update && !del && !cachePath && !settings) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
|
|
@ -195,6 +203,21 @@ export function ModelRowMenu({
|
|||
sideOffset={2}
|
||||
className="unsloth-plus-menu menu-flat-destructive w-48"
|
||||
>
|
||||
{settings && (
|
||||
<DropdownMenuItem
|
||||
onSelect={(e) => {
|
||||
e.stopPropagation();
|
||||
settings.onOpen();
|
||||
}}
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Settings02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-icon"
|
||||
/>
|
||||
<span>Settings</span>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{pin && (
|
||||
<DropdownMenuItem
|
||||
onSelect={(e) => {
|
||||
|
|
|
|||
|
|
@ -49,6 +49,22 @@ export interface ModelPickTarget {
|
|||
displayName: string;
|
||||
ggufVariant?: string | null;
|
||||
isGguf: boolean;
|
||||
/**
|
||||
* Whether an OpenAI-compatible request can actually load this model.
|
||||
*
|
||||
* Not the same as isGguf: local_model_resolver skips Ollama's scanner, so an
|
||||
* Ollama GGUF is never in the auto-switch index and mirroring its settings would
|
||||
* advertise a load that cannot happen. Defaults to isGguf when unknown.
|
||||
*/
|
||||
apiLoadable?: boolean;
|
||||
/**
|
||||
* Identity the saved settings are keyed by, when that is not what loads.
|
||||
*
|
||||
* A repo cached outside the active HF cache loads by snapshot path, while the
|
||||
* picker and the auto-switch index key its settings by repo id, so saving by the
|
||||
* path would strand them. Probes that must open the model keep using `id`.
|
||||
*/
|
||||
configId?: string;
|
||||
meta: ModelSelectorChangeMeta;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@
|
|||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { useMemo } from "react";
|
||||
import { gpuFieldsSignature } from "../model-config/apply-per-model-config";
|
||||
import { modelConfigInstanceKey } from "../model-config/config-signature";
|
||||
import {
|
||||
isOllamaLinkPath,
|
||||
isStandaloneGgufPath,
|
||||
} from "../model-config/model-identity";
|
||||
import type { PerModelConfig } from "../model-config/per-model-config";
|
||||
import { ModelConfigPage } from "./model-config-page";
|
||||
import type { ModelPickTarget } from "./model-selector/types";
|
||||
|
|
@ -28,30 +32,6 @@ function leafName(id: string): string {
|
|||
return separator >= 0 ? trimmed.slice(separator + 1) : trimmed;
|
||||
}
|
||||
|
||||
function hashString(value: string): number {
|
||||
let hash = 5381;
|
||||
for (let i = 0; i < value.length; i += 1) {
|
||||
hash = (Math.imul(hash, 33) ^ value.charCodeAt(i)) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
function configSignature(config: PerModelConfig): string {
|
||||
return [
|
||||
config.customContextLength ?? "",
|
||||
config.maxSeqLength ?? "",
|
||||
config.kvCacheDtype ?? "",
|
||||
config.speculativeType ?? "",
|
||||
config.specDraftNMax ?? "",
|
||||
config.nParallel ?? "",
|
||||
config.tensorParallel ? "1" : "0",
|
||||
config.chatTemplateOverride == null
|
||||
? ""
|
||||
: `${config.chatTemplateOverride.length}:${hashString(config.chatTemplateOverride)}`,
|
||||
gpuFieldsSignature(config),
|
||||
].join("|");
|
||||
}
|
||||
|
||||
export function SidebarModelConfig({
|
||||
modelId,
|
||||
ggufVariant,
|
||||
|
|
@ -61,27 +41,38 @@ export function SidebarModelConfig({
|
|||
loadedConfig,
|
||||
onReload,
|
||||
}: SidebarModelConfigProps) {
|
||||
// A standalone .gguf has no quant to choose between, but the loader labels it
|
||||
// from its filename (llama_cpp falls back to _extract_quant_label when the load
|
||||
// named no variant) and /status echoes that as gguf_variant. Keying settings by
|
||||
// it would write "<path>:Q4_K_M" while the Hub row, the picker and the backfill
|
||||
// all use the bare path, so the same file would carry two configs. The
|
||||
// auto-switch lookup reads the bare path first, so the sidebar's entry would
|
||||
// never be the one an API load applies. Same rule as settingsGgufVariantForRow.
|
||||
const settingsGgufVariant = isStandaloneGgufPath(modelId) ? null : ggufVariant;
|
||||
const target = useMemo<ModelPickTarget>(() => {
|
||||
const leaf = leafName(modelId);
|
||||
return {
|
||||
id: modelId,
|
||||
displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf,
|
||||
ggufVariant,
|
||||
ggufVariant: settingsGgufVariant,
|
||||
isGguf,
|
||||
// An Ollama blob loads through a link dir the auto-switch resolver skips,
|
||||
// so its settings must not be mirrored as if the API could load it.
|
||||
apiLoadable: isGguf && !isOllamaLinkPath(modelId),
|
||||
meta: {
|
||||
source: "local",
|
||||
isLora: false,
|
||||
ggufVariant: ggufVariant ?? undefined,
|
||||
ggufVariant: settingsGgufVariant ?? undefined,
|
||||
isGguf,
|
||||
isDownloaded: true,
|
||||
contextLength: nativeContextLength,
|
||||
},
|
||||
};
|
||||
}, [modelId, ggufVariant, isGguf, nativeContextLength]);
|
||||
}, [modelId, ggufVariant, settingsGgufVariant, isGguf, nativeContextLength]);
|
||||
|
||||
return (
|
||||
<ModelConfigPage
|
||||
key={`${modelId}::${ggufVariant ?? ""}::${configSignature(loadedConfig)}`}
|
||||
key={modelConfigInstanceKey(modelId, settingsGgufVariant, loadedConfig)}
|
||||
target={target}
|
||||
onRun={onReload}
|
||||
loadedConfig={loadedConfig}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,17 @@ export {
|
|||
type NumericValueInputHandle,
|
||||
snapToStep,
|
||||
} from "./components/numeric-value-input";
|
||||
export { ModelConfigPage } from "./components/model-config-page";
|
||||
export { SidebarModelConfig } from "./components/sidebar-model-config";
|
||||
export type { ModelPickTarget } from "./components/model-selector/types";
|
||||
export {
|
||||
fetchModelOverrides,
|
||||
modelOverrideKey,
|
||||
putModelOverride,
|
||||
syncModelOverride,
|
||||
type ApiModelOverride,
|
||||
type ApiModelOverrides,
|
||||
} from "./api/model-overrides";
|
||||
export {
|
||||
useActiveModelConfig,
|
||||
} from "./hooks/use-active-model-config";
|
||||
|
|
@ -26,6 +36,7 @@ export type {
|
|||
ModelOption,
|
||||
ModelSelectorChangeMeta,
|
||||
} from "./components/model-selector";
|
||||
export { modelConfigInstanceKey } from "./model-config/config-signature";
|
||||
export {
|
||||
applyModelLoadConfigToRuntime,
|
||||
applyPerModelConfigToRuntime,
|
||||
|
|
|
|||
|
|
@ -10,12 +10,17 @@ import {
|
|||
reconcilePersistedGpuIds,
|
||||
useChatRuntimeStore,
|
||||
} from "@/features/chat";
|
||||
// Lives in its own module so hosts that only need the signature (the editor keys)
|
||||
// can import it without pulling the chat runtime store in with it.
|
||||
import { gpuFieldsSignature } from "./config-signature";
|
||||
import {
|
||||
DEFAULT_PER_MODEL_CONFIG,
|
||||
type PerModelConfig,
|
||||
normalizeMaxSeqLength,
|
||||
} from "./per-model-config";
|
||||
|
||||
export { gpuFieldsSignature };
|
||||
|
||||
function cleanTemplate(value: string | null | undefined): string | null {
|
||||
return value?.trim() ? value : null;
|
||||
}
|
||||
|
|
@ -111,20 +116,6 @@ export function perModelConfigsEqual(
|
|||
);
|
||||
}
|
||||
|
||||
// Serialize the per-model GPU knobs with the same "absent == default"
|
||||
// coalescing the store applies: mode auto/absent, gpuLayers Auto (< 0) /
|
||||
// absent, nCpuMoe 0 / absent, and the GPU pick (null / absent = all GPUs).
|
||||
export function gpuFieldsSignature(config: PerModelConfig): string {
|
||||
return [
|
||||
config.gpuMemoryMode ?? "auto",
|
||||
config.gpuLayers == null || config.gpuLayers < 0 ? -1 : config.gpuLayers,
|
||||
config.nCpuMoe ?? 0,
|
||||
config.selectedGpuIds == null
|
||||
? "all"
|
||||
: [...config.selectedGpuIds].sort((a, b) => a - b).join(","),
|
||||
].join("|");
|
||||
}
|
||||
|
||||
function gpuFieldsEqual(a: PerModelConfig, b: PerModelConfig): boolean {
|
||||
return gpuFieldsSignature(a) === gpuFieldsSignature(b);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,77 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Identity of one ModelConfigPage editor instance.
|
||||
//
|
||||
// ModelConfigPage seeds its editable state from `loadedConfig` in a useState
|
||||
// initializer, so it reads that prop exactly once per mounted instance. A host
|
||||
// that opens the page before /api/inference/status has hydrated, or while the
|
||||
// target is still loading, gets `loadedConfig` null first and the live config a
|
||||
// moment later; without the config in the React key the same instance survives
|
||||
// that flip and keeps showing the saved/default values for a model that is
|
||||
// running with something else, which Apply then writes back over it.
|
||||
|
||||
import type { PerModelConfig } from "./per-model-config";
|
||||
|
||||
// Serialize the per-model GPU knobs with the same "absent == default"
|
||||
// coalescing the store applies: mode auto/absent, gpuLayers Auto (< 0) /
|
||||
// absent, nCpuMoe 0 / absent, and the GPU pick (null / absent = all GPUs).
|
||||
export function gpuFieldsSignature(config: PerModelConfig): string {
|
||||
return [
|
||||
config.gpuMemoryMode ?? "auto",
|
||||
config.gpuLayers == null || config.gpuLayers < 0 ? -1 : config.gpuLayers,
|
||||
config.nCpuMoe ?? 0,
|
||||
config.selectedGpuIds == null
|
||||
? "all"
|
||||
: [...config.selectedGpuIds].sort((a, b) => a - b).join(","),
|
||||
].join("|");
|
||||
}
|
||||
|
||||
function hashString(value: string): number {
|
||||
let hash = 5381;
|
||||
for (let i = 0; i < value.length; i += 1) {
|
||||
hash = (Math.imul(hash, 33) ^ value.charCodeAt(i)) >>> 0;
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
/**
|
||||
* Signature of the live config an editor was seeded from.
|
||||
*
|
||||
* `null` (no live config, because the model is not resident or status has not
|
||||
* answered yet) is deliberately its own value, distinct from every real config:
|
||||
* the arrival of the live config is exactly the transition that has to remount.
|
||||
*/
|
||||
export function loadedConfigSignature(
|
||||
config: PerModelConfig | null | undefined,
|
||||
): string {
|
||||
if (!config) {
|
||||
return "none";
|
||||
}
|
||||
return [
|
||||
config.customContextLength ?? "",
|
||||
config.maxSeqLength ?? "",
|
||||
config.kvCacheDtype ?? "",
|
||||
config.speculativeType ?? "",
|
||||
config.specDraftNMax ?? "",
|
||||
config.nParallel ?? "",
|
||||
config.tensorParallel ? "1" : "0",
|
||||
config.chatTemplateOverride == null
|
||||
? ""
|
||||
: `${config.chatTemplateOverride.length}:${hashString(config.chatTemplateOverride)}`,
|
||||
gpuFieldsSignature(config),
|
||||
].join("|");
|
||||
}
|
||||
|
||||
/**
|
||||
* React key for one ModelConfigPage instance. Every host mounts it under this so
|
||||
* they agree on when the editor is re-seeded: on a different model, a different
|
||||
* quant, or a change in the live config it is meant to be showing.
|
||||
*/
|
||||
export function modelConfigInstanceKey(
|
||||
modelId: string,
|
||||
ggufVariant: string | null | undefined,
|
||||
loadedConfig: PerModelConfig | null | undefined,
|
||||
): string {
|
||||
return `${modelId}::${ggufVariant ?? ""}::${loadedConfigSignature(loadedConfig)}`;
|
||||
}
|
||||
|
|
@ -1,15 +1,21 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Straight from the module rather than the hub barrel, which also re-exports the
|
||||
// download manager and its React components: these are pure string helpers, and
|
||||
// pulling the barrel in puts every one of those in the way of loading them.
|
||||
import {
|
||||
normalizeGgufVariantIdentity,
|
||||
normalizeModelIdentity,
|
||||
} from "@/features/hub";
|
||||
} from "@/features/hub/lib/model-identity";
|
||||
|
||||
export {
|
||||
isNativeFileLabel,
|
||||
isOllamaLinkPath,
|
||||
isStandaloneGgufPath,
|
||||
normalizeGgufVariantIdentity,
|
||||
normalizeModelIdentity,
|
||||
} from "@/features/hub";
|
||||
} from "@/features/hub/lib/model-identity";
|
||||
|
||||
const MODEL_STORAGE_KEY_PREFIX = "v2:";
|
||||
|
||||
|
|
@ -67,3 +73,89 @@ export function ggufVariantFromStorageKey(key: string): string | null {
|
|||
const separator = key.lastIndexOf("::");
|
||||
return separator >= 0 ? key.slice(separator + 2) : null;
|
||||
}
|
||||
|
||||
// Mirrors split_quant_suffix in studio/backend/utils/openai_auto_switch_settings.py.
|
||||
// The bpw modifier ("IQ4_XS-3.53bpw") is optional: the backend label helpers disagree.
|
||||
const BPW_SUFFIX = /-[0-9]+(?:\.[0-9]+)?bpw$/i;
|
||||
// One source for the anchored test and the scan below, so they cannot drift apart.
|
||||
// Mirrors _GGUF_QUANT_RE in studio/backend/hub/utils/gguf.py.
|
||||
const QUANT_TOKEN_SOURCE =
|
||||
"(UD-)?(MXFP[0-9]+(?:_[A-Z0-9]+)*|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?|TQ[0-9]+_[0-9]+|Q[0-9]+_K_[A-Z]+|Q[0-9]+_[0-9]+|Q[0-9]+_K|BF16|F16|F32)";
|
||||
const KNOWN_QUANT = new RegExp(`^${QUANT_TOKEN_SOURCE}$`, "i");
|
||||
const QUANT_TOKEN = new RegExp(QUANT_TOKEN_SOURCE, "gi");
|
||||
const MAX_QUANT_SUFFIX_LEN = 64;
|
||||
// Mirrors _GGUF_SPLIT_SUFFIX_RE in studio/backend/hub/utils/gguf.py.
|
||||
const GGUF_SPLIT_SUFFIX = /-[0-9]{3,}-of-[0-9]{3,}/gi;
|
||||
const BACKSLASHES = /\\/g;
|
||||
// A float precision only labels a file when nothing sharper does, matching the
|
||||
// backend's _select_quant_match.
|
||||
const FLOAT_PRECISION_QUANTS: ReadonlySet<string> = new Set([
|
||||
"BF16",
|
||||
"F16",
|
||||
"F32",
|
||||
]);
|
||||
|
||||
/** Mirrors _gguf_stem in studio/backend/hub/utils/gguf.py, for a bare filename. */
|
||||
function ggufStem(filename: string): string {
|
||||
const dot = filename.lastIndexOf(".");
|
||||
const withoutExtension = dot >= 0 ? filename.slice(0, dot) : filename;
|
||||
return withoutExtension.replace(GGUF_SPLIT_SUFFIX, "").trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirrors extract_quant_label in studio/backend/hub/utils/gguf.py, for a bare
|
||||
* filename. The parent-directory pass there cannot fire on a basename, so this
|
||||
* is the stem's own quant token or, failing that, the stem itself.
|
||||
*/
|
||||
function ggufQuantLabel(filename: string): string {
|
||||
const stem = ggufStem(filename);
|
||||
let fallback: RegExpExecArray | null = null;
|
||||
for (const match of stem.matchAll(QUANT_TOKEN)) {
|
||||
if (FLOAT_PRECISION_QUANTS.has(match[2].toUpperCase())) {
|
||||
fallback ??= match;
|
||||
continue;
|
||||
}
|
||||
return `${match[1] ?? ""}${match[2]}`;
|
||||
}
|
||||
if (fallback) {
|
||||
return `${fallback[1] ?? ""}${fallback[2]}`;
|
||||
}
|
||||
return stem || "gguf";
|
||||
}
|
||||
|
||||
/**
|
||||
* `[head, quant]` for a `head:QUANT` key, or null when the colon is not one.
|
||||
*
|
||||
* The suffix must look like a real quant, so an ordinary colon in a POSIX filename
|
||||
* ("/models/foo:bar.gguf") and a Windows drive letter are left alone.
|
||||
*/
|
||||
export function splitQuantSuffix(value: string): [string, string] | null {
|
||||
const separator = value.lastIndexOf(":");
|
||||
if (separator <= 0 || separator === value.length - 1) {
|
||||
return null;
|
||||
}
|
||||
const head = value.slice(0, separator);
|
||||
const tail = value.slice(separator + 1);
|
||||
if (tail.includes("/") || tail.includes("\\")) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
tail.length <= MAX_QUANT_SUFFIX_LEN &&
|
||||
KNOWN_QUANT.test(tail.replace(BPW_SUFFIX, ""))
|
||||
) {
|
||||
return [head, tail];
|
||||
}
|
||||
// A .gguf with no recognizable quant is labelled by its stem, so
|
||||
// "/models/CustomModel.gguf:custommodel" exists; a non-.gguf head is a plain colon.
|
||||
if (!head.toLowerCase().endsWith(".gguf")) {
|
||||
return null;
|
||||
}
|
||||
// The suffix has to be that exact label, as the backend requires. A colon is legal
|
||||
// in a POSIX filename, so "/models/llama.gguf:Bar.gguf" and its lowercase sibling
|
||||
// are two real files: reading the suffix as a variant folds them onto one key and
|
||||
// strands one file's settings, since the variant half is stored lowercased.
|
||||
const filename = head.replace(BACKSLASHES, "/").split("/").pop() ?? head;
|
||||
return tail.toLowerCase() === ggufQuantLabel(filename).toLowerCase()
|
||||
? [head, tail]
|
||||
: null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -225,6 +225,7 @@ function serializedMapEntrySize(key: string, value: StoredMap[string]): number {
|
|||
function deleteOldestEvictableEntry(
|
||||
map: StoredMap,
|
||||
protectedKeys?: ReadonlySet<string>,
|
||||
evicted?: string[],
|
||||
): { key: string; value: StoredMap[string] } | null {
|
||||
for (const key of Object.keys(map)) {
|
||||
// Never evict a future-schema entry an older client cannot interpret.
|
||||
|
|
@ -236,6 +237,7 @@ function deleteOldestEvictableEntry(
|
|||
}
|
||||
const value = map[key];
|
||||
delete map[key];
|
||||
evicted?.push(key);
|
||||
return { key, value };
|
||||
}
|
||||
return null;
|
||||
|
|
@ -244,17 +246,18 @@ function deleteOldestEvictableEntry(
|
|||
function enforceStorageBudget(
|
||||
map: StoredMap,
|
||||
protectedKeys?: ReadonlySet<string>,
|
||||
evicted?: string[],
|
||||
): boolean {
|
||||
let entryCount = Object.keys(map).length;
|
||||
while (entryCount > MAX_ENTRIES) {
|
||||
if (!deleteOldestEvictableEntry(map, protectedKeys)) {
|
||||
if (!deleteOldestEvictableEntry(map, protectedKeys, evicted)) {
|
||||
return false;
|
||||
}
|
||||
entryCount -= 1;
|
||||
}
|
||||
let bytes = serializedMapSize(map);
|
||||
while (bytes > MAX_PER_MODEL_CONFIG_STORAGE_BYTES) {
|
||||
const removed = deleteOldestEvictableEntry(map, protectedKeys);
|
||||
const removed = deleteOldestEvictableEntry(map, protectedKeys, evicted);
|
||||
if (!removed) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -426,7 +429,10 @@ function writeMap(map: StoredMap): boolean {
|
|||
}
|
||||
}
|
||||
|
||||
function warnDroppedFields(raw: Record<string, unknown>, version: number): void {
|
||||
function warnDroppedFields(
|
||||
raw: Record<string, unknown>,
|
||||
version: number,
|
||||
): void {
|
||||
if (!import.meta.env?.DEV) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -446,7 +452,8 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
|
|||
typeof partial.speculativeType === "string"
|
||||
? canonicalizeSpeculativeType(partial.speculativeType)
|
||||
: null;
|
||||
const speculativeType = rawSpecType ?? DEFAULT_PER_MODEL_CONFIG.speculativeType;
|
||||
const speculativeType =
|
||||
rawSpecType ?? DEFAULT_PER_MODEL_CONFIG.speculativeType;
|
||||
const specDraftNMax =
|
||||
speculativeType != null &&
|
||||
MTP_SPECULATIVE_TYPES.has(speculativeType) &&
|
||||
|
|
@ -486,6 +493,15 @@ function normalizeV1(partial: RawConfig): PerModelConfig {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A config in the exact shape storage keeps it in: the UI carries sentinels storage
|
||||
* does not, notably Speculative Decoding "auto" which canonicalizes to null, so a
|
||||
* config still being edited reads as non-default when it is not.
|
||||
*/
|
||||
export function normalizePerModelConfig(raw: unknown): PerModelConfig {
|
||||
return normalize(raw);
|
||||
}
|
||||
|
||||
function normalize(raw: unknown): PerModelConfig {
|
||||
if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
|
||||
return normalizeV1({});
|
||||
|
|
@ -634,6 +650,12 @@ export function savePerModelConfig(
|
|||
modelId: string,
|
||||
ggufVariant: string | null | undefined,
|
||||
config: PerModelConfig,
|
||||
/**
|
||||
* Receives models dropped to stay inside the storage budget. Eviction is silent
|
||||
* and still reports success, so without this their server-side overrides would
|
||||
* keep being applied with nothing in the UI able to forget them.
|
||||
*/
|
||||
evicted?: { modelId: string; ggufVariant: string | null }[],
|
||||
): boolean {
|
||||
if (
|
||||
typeof config.chatTemplateOverride === "string" &&
|
||||
|
|
@ -657,10 +679,54 @@ export function savePerModelConfig(
|
|||
const [key] = storageKeysForModelVariant(modelId, ggufVariant);
|
||||
deleteConfigEntriesForModelVariant(map, modelId, ggufVariant);
|
||||
map[key] = toStoredConfig(normalized);
|
||||
if (!enforceStorageBudget(map, new Set([key]))) {
|
||||
const evictedKeys: string[] = [];
|
||||
if (!enforceStorageBudget(map, new Set([key]), evictedKeys)) {
|
||||
return false;
|
||||
}
|
||||
return writeMap(map);
|
||||
const written = writeMap(map);
|
||||
if (written && evicted) {
|
||||
for (const evictedKey of evictedKeys) {
|
||||
const id = modelIdFromStorageKey(evictedKey);
|
||||
if (!id) {
|
||||
continue;
|
||||
}
|
||||
const variant = ggufVariantFromStorageKey(evictedKey);
|
||||
evicted.push({ modelId: id, ggufVariant: variant ? variant : null });
|
||||
}
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
/** Every saved per-model config, decoded back to the ids it was keyed by. */
|
||||
export function listPerModelConfigs(): {
|
||||
modelId: string;
|
||||
ggufVariant: string | null;
|
||||
config: PerModelConfig;
|
||||
}[] {
|
||||
const out: {
|
||||
modelId: string;
|
||||
ggufVariant: string | null;
|
||||
config: PerModelConfig;
|
||||
}[] = [];
|
||||
for (const [key, raw] of Object.entries(readMap())) {
|
||||
const modelId = modelIdFromStorageKey(key);
|
||||
if (!modelId) {
|
||||
continue;
|
||||
}
|
||||
// Never report a future-schema record: loadPerModelConfig refuses to apply one
|
||||
// and eviction refuses to drop one, so the backfill would persist this client's
|
||||
// partial reading and let an API load apply what it will not apply locally.
|
||||
if (storedConfigVersion(raw) > STORAGE_SCHEMA_VERSION) {
|
||||
continue;
|
||||
}
|
||||
const variant = ggufVariantFromStorageKey(key);
|
||||
out.push({
|
||||
modelId,
|
||||
ggufVariant: variant ? variant : null,
|
||||
config: normalize(raw),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function deletePerModelConfig(
|
||||
|
|
|
|||
|
|
@ -1,578 +0,0 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
ActivityIcon,
|
||||
ChevronDownIcon,
|
||||
CircleIcon,
|
||||
PowerOffIcon,
|
||||
RefreshCwIcon,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
type ReactElement,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import {
|
||||
getApiMonitor,
|
||||
getApiMonitorEntry,
|
||||
getInferenceStatus,
|
||||
unloadModel,
|
||||
} from "../../chat/api/chat-api";
|
||||
import { resolveInferenceCheckpointId } from "../../chat/lib/apply-inference-status-to-store";
|
||||
import { useChatRuntimeStore } from "../../chat/stores/chat-runtime-store";
|
||||
import type { ApiMonitorEntry, ApiMonitorResponse } from "../../chat/types/api";
|
||||
|
||||
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
|
||||
const V1_PREFIX_RE = /^\/v1\//;
|
||||
const PAGE_SIZE = 5;
|
||||
|
||||
function formatTime(value: number): string {
|
||||
return new Date(value * 1000).toLocaleTimeString([], {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
}
|
||||
|
||||
function formatDuration(value?: number | null): string {
|
||||
if (value == null) {
|
||||
return "Running";
|
||||
}
|
||||
if (value < 1000) {
|
||||
return `${value} ms`;
|
||||
}
|
||||
return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`;
|
||||
}
|
||||
|
||||
function formatTokens(entry: ApiMonitorEntry): string {
|
||||
if (entry.total_tokens != null) {
|
||||
return `${entry.total_tokens.toLocaleString()} tokens`;
|
||||
}
|
||||
if (entry.prompt_tokens != null || entry.completion_tokens != null) {
|
||||
const prompt = entry.prompt_tokens ?? 0;
|
||||
const completion = entry.completion_tokens ?? 0;
|
||||
return `${(prompt + completion).toLocaleString()} tokens`;
|
||||
}
|
||||
return "Tokens pending";
|
||||
}
|
||||
|
||||
function compactEndpoint(endpoint: string): string {
|
||||
return endpoint
|
||||
.replace(API_INFERENCE_PREFIX_RE, "/api")
|
||||
.replace(V1_PREFIX_RE, "/");
|
||||
}
|
||||
|
||||
function statusTone(status: ApiMonitorEntry["status"]): string {
|
||||
if (status === "running") {
|
||||
return "text-emerald-500";
|
||||
}
|
||||
if (status === "error") {
|
||||
return "text-destructive";
|
||||
}
|
||||
if (status === "cancelled") {
|
||||
return "text-amber-500";
|
||||
}
|
||||
return "text-muted-foreground";
|
||||
}
|
||||
|
||||
function UsageBar({ value }: { value?: number | null }): ReactElement | null {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
const pct = Math.max(0, Math.min(100, Math.round(value * 100)));
|
||||
return (
|
||||
<div className="mt-2 h-1.5 w-full overflow-hidden rounded-full bg-muted">
|
||||
<div
|
||||
className="h-full rounded-full bg-control-accent"
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function isLifecycle(entry: ApiMonitorEntry): boolean {
|
||||
return entry.kind === "lifecycle";
|
||||
}
|
||||
|
||||
function lifecycleLabel(entry: ApiMonitorEntry): string {
|
||||
if (entry.event === "unload") {
|
||||
return entry.reason === "idle" ? "Model unloaded (idle)" : "Model unloaded";
|
||||
}
|
||||
if (entry.event === "download") {
|
||||
if (entry.status === "running") {
|
||||
const pct = entry.progress;
|
||||
return typeof pct === "number"
|
||||
? `Downloading model (${Math.round(pct)}%)`
|
||||
: "Downloading model";
|
||||
}
|
||||
if (entry.status === "completed") return "Model downloaded";
|
||||
// A cancel is deliberate, so saying it failed misreads the user's own action.
|
||||
return entry.status === "cancelled"
|
||||
? "Model download cancelled"
|
||||
: "Model download failed";
|
||||
}
|
||||
if (entry.status === "running") {
|
||||
return "Loading model";
|
||||
}
|
||||
if (entry.status === "completed") {
|
||||
return "Model loaded";
|
||||
}
|
||||
return "Model load failed";
|
||||
}
|
||||
|
||||
// Load/unload rows: label, model and time. No prompt or detail, so nothing to expand.
|
||||
function LifecycleEntry({ entry }: { entry: ApiMonitorEntry }): ReactElement {
|
||||
return (
|
||||
<article className="min-w-0 rounded-lg border border-border/70 bg-muted/25">
|
||||
<div className="flex w-full min-w-0 items-start justify-between gap-3 p-3">
|
||||
<div className="min-w-0">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<ActivityIcon
|
||||
className={cn("size-3.5 shrink-0", statusTone(entry.status))}
|
||||
/>
|
||||
<span className="truncate text-xs font-medium">
|
||||
{lifecycleLabel(entry)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1 truncate text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</div>
|
||||
</div>
|
||||
<div className="shrink-0 text-right text-ui-11 text-muted-foreground">
|
||||
<div>{formatTime(entry.started_at)}</div>
|
||||
{entry.event === "load" || entry.event === "download" ? (
|
||||
<div>{formatDuration(entry.duration_ms)}</div>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
||||
function MonitorEntry({
|
||||
entry,
|
||||
detail,
|
||||
expanded,
|
||||
loading,
|
||||
onToggle,
|
||||
}: {
|
||||
entry: ApiMonitorEntry;
|
||||
detail?: ApiMonitorEntry;
|
||||
expanded: boolean;
|
||||
loading: boolean;
|
||||
onToggle: () => void;
|
||||
}): ReactElement {
|
||||
const hasCurrentDetail =
|
||||
detail &&
|
||||
detail.status === entry.status &&
|
||||
detail.updated_at >= entry.updated_at;
|
||||
const prompt = detail?.prompt ?? entry.prompt_preview;
|
||||
const replyText = hasCurrentDetail
|
||||
? detail.error ?? detail.reply ?? entry.error ?? entry.reply_preview
|
||||
: entry.error ?? entry.reply_preview;
|
||||
const reply = replyText || (entry.status === "running" ? "Waiting..." : "No reply");
|
||||
|
||||
return (
|
||||
<article className="min-w-0 rounded-lg border border-border/70 bg-background">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggle}
|
||||
className="flex w-full min-w-0 items-start justify-between gap-3 p-3 text-left"
|
||||
aria-expanded={expanded}
|
||||
>
|
||||
<div className="min-w-0">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<CircleIcon
|
||||
className={cn("size-2.5 fill-current", statusTone(entry.status))}
|
||||
/>
|
||||
<span className="truncate text-xs font-medium">
|
||||
{compactEndpoint(entry.endpoint)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1 truncate text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</div>
|
||||
<div className="mt-2 line-clamp-2 whitespace-pre-wrap break-words text-xs text-muted-foreground">
|
||||
{entry.error ||
|
||||
entry.reply_preview ||
|
||||
entry.prompt_preview ||
|
||||
(entry.status === "running" ? "Waiting..." : "No preview")}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-start gap-2 text-right text-ui-11 text-muted-foreground">
|
||||
<div>
|
||||
<div>{formatTime(entry.started_at)}</div>
|
||||
<div>{formatDuration(entry.duration_ms)}</div>
|
||||
</div>
|
||||
<ChevronDownIcon
|
||||
className={cn(
|
||||
"mt-0.5 size-3.5 transition-transform",
|
||||
expanded && "rotate-180",
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
{expanded ? (
|
||||
<div className="border-t border-border/60 p-3 pt-2">
|
||||
<div className="grid gap-2">
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between gap-2 text-ui-10 font-semibold uppercase text-muted-foreground">
|
||||
<span>Prompt</span>
|
||||
{entry.prompt_truncated && !detail ? <span>Preview</span> : null}
|
||||
</div>
|
||||
<pre className="max-h-44 overflow-auto whitespace-pre-wrap break-words rounded-md bg-muted/45 p-2 text-xs leading-5">
|
||||
{loading && !detail ? "Loading..." : prompt || "No prompt text"}
|
||||
</pre>
|
||||
</div>
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between gap-2 text-ui-10 font-semibold uppercase text-muted-foreground">
|
||||
<span>Reply</span>
|
||||
{entry.reply_truncated && !detail ? <span>Preview</span> : null}
|
||||
</div>
|
||||
<pre className="max-h-44 overflow-auto whitespace-pre-wrap break-words rounded-md bg-muted/45 p-2 text-xs leading-5">
|
||||
{loading && !detail ? "Loading..." : reply}
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-3 text-ui-11 text-muted-foreground">
|
||||
{formatTokens(entry)}
|
||||
{entry.context_length ? (
|
||||
<> / {entry.context_length.toLocaleString()} context</>
|
||||
) : null}
|
||||
<UsageBar value={entry.context_usage} />
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
||||
export function ApiMonitorConsole(): ReactElement {
|
||||
const [data, setData] = useState<ApiMonitorResponse | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [unloading, setUnloading] = useState(false);
|
||||
const [expandedIds, setExpandedIds] = useState<Set<string>>(() => new Set());
|
||||
const [details, setDetails] = useState<Record<string, ApiMonitorEntry>>({});
|
||||
const [loadingDetails, setLoadingDetails] = useState<Set<string>>(
|
||||
() => new Set(),
|
||||
);
|
||||
const loadingDetailsRef = useRef<Set<string>>(new Set());
|
||||
const detailsRef = useRef<Record<string, ApiMonitorEntry>>({});
|
||||
|
||||
const loadMonitor = useCallback(async (): Promise<void> => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
setData(await getApiMonitor());
|
||||
setError(null);
|
||||
} catch (err: unknown) {
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
// /unload matches on the internal id, which the monitor omits, so read it from status.
|
||||
const unloadActiveModel = useCallback(async (): Promise<void> => {
|
||||
setUnloading(true);
|
||||
try {
|
||||
const status = await getInferenceStatus();
|
||||
const checkpoint = resolveInferenceCheckpointId(status);
|
||||
if (!checkpoint) {
|
||||
setError(null);
|
||||
return;
|
||||
}
|
||||
await unloadModel({ model_path: checkpoint });
|
||||
// Same as the chat eject flow: the store still holds the freed checkpoint.
|
||||
useChatRuntimeStore.getState().clearCheckpoint();
|
||||
setError(null);
|
||||
await loadMonitor();
|
||||
} catch (err: unknown) {
|
||||
setError(err instanceof Error ? err.message : "Failed to unload the model");
|
||||
} finally {
|
||||
setUnloading(false);
|
||||
}
|
||||
}, [loadMonitor]);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
let timer: number | undefined;
|
||||
|
||||
function schedule(): void {
|
||||
timer = window.setTimeout(poll, 1500);
|
||||
}
|
||||
|
||||
function poll(): void {
|
||||
getApiMonitor()
|
||||
.then((next) => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
setData(next);
|
||||
setError(null);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) {
|
||||
schedule();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
poll();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer !== undefined) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const statusLabel = data?.status ?? "idle";
|
||||
const hasActive = (data?.active_requests ?? 0) > 0;
|
||||
const entries = useMemo(() => data?.entries ?? [], [data]);
|
||||
|
||||
// Page 1 tracks the live list; paging back freezes the id order so history holds still.
|
||||
const [page, setPage] = useState(0);
|
||||
const [frozenIds, setFrozenIds] = useState<string[] | null>(null);
|
||||
const byId = useMemo(
|
||||
() => new Map(entries.map((entry) => [entry.id, entry])),
|
||||
[entries],
|
||||
);
|
||||
const ordered = useMemo(() => {
|
||||
if (frozenIds === null) {
|
||||
return entries;
|
||||
}
|
||||
return frozenIds.flatMap((id) => {
|
||||
const entry = byId.get(id);
|
||||
return entry ? [entry] : [];
|
||||
});
|
||||
}, [byId, entries, frozenIds]);
|
||||
const pageCount = Math.max(1, Math.ceil(ordered.length / PAGE_SIZE));
|
||||
const pageIndex = Math.min(page, pageCount - 1);
|
||||
const visible = ordered.slice(
|
||||
pageIndex * PAGE_SIZE,
|
||||
pageIndex * PAGE_SIZE + PAGE_SIZE,
|
||||
);
|
||||
const newerCount =
|
||||
frozenIds === null
|
||||
? 0
|
||||
: entries.filter((entry) => !frozenIds.includes(entry.id)).length;
|
||||
|
||||
const goToPage = useCallback(
|
||||
(next: number): void => {
|
||||
if (next <= 0) {
|
||||
setFrozenIds(null);
|
||||
setPage(0);
|
||||
return;
|
||||
}
|
||||
// Freeze on the way off page 1 so the history under the cursor holds still.
|
||||
setFrozenIds((prev) => prev ?? entries.map((entry) => entry.id));
|
||||
setPage(next);
|
||||
},
|
||||
[entries],
|
||||
);
|
||||
|
||||
const loadDetail = useCallback(
|
||||
(id: string): void => {
|
||||
if (loadingDetailsRef.current.has(id)) {
|
||||
return;
|
||||
}
|
||||
loadingDetailsRef.current.add(id);
|
||||
setLoadingDetails((prev) => new Set(prev).add(id));
|
||||
getApiMonitorEntry(id)
|
||||
.then((entry) => {
|
||||
setDetails((prev) => {
|
||||
const next = { ...prev, [id]: entry };
|
||||
detailsRef.current = next;
|
||||
return next;
|
||||
});
|
||||
})
|
||||
.catch(() => {
|
||||
setDetails((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[id];
|
||||
detailsRef.current = next;
|
||||
return next;
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
loadingDetailsRef.current.delete(id);
|
||||
setLoadingDetails((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(id);
|
||||
return next;
|
||||
});
|
||||
});
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const toggleEntry = useCallback(
|
||||
(entry: ApiMonitorEntry): void => {
|
||||
setExpandedIds((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(entry.id)) {
|
||||
next.delete(entry.id);
|
||||
} else {
|
||||
next.add(entry.id);
|
||||
loadDetail(entry.id);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
},
|
||||
[loadDetail],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
// Only rows on screen: an expanded row on another page would keep polling.
|
||||
for (const entry of visible) {
|
||||
if (isLifecycle(entry) || !expandedIds.has(entry.id)) {
|
||||
continue;
|
||||
}
|
||||
const cached = detailsRef.current[entry.id];
|
||||
if (!cached || cached.status !== entry.status || entry.status === "running") {
|
||||
loadDetail(entry.id);
|
||||
}
|
||||
}
|
||||
}, [visible, expandedIds, loadDetail]);
|
||||
|
||||
return (
|
||||
<section className="flex min-w-0 flex-col rounded-lg border border-border/70 bg-background">
|
||||
<div className="flex min-w-0 items-start justify-between gap-3 border-b border-border/60 px-4 py-3">
|
||||
<div className="flex min-w-0 gap-3">
|
||||
<div className="relative mt-0.5 flex size-8 shrink-0 items-center justify-center rounded-md border border-border/70 bg-muted/40">
|
||||
<ActivityIcon className="size-4 text-foreground" />
|
||||
{hasActive ? (
|
||||
<span className="absolute right-1 top-1 size-2 rounded-full bg-emerald-500" />
|
||||
) : null}
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
<h2 className="text-sm font-semibold text-foreground">
|
||||
API monitor
|
||||
</h2>
|
||||
<p className="truncate text-xs text-muted-foreground">
|
||||
{data?.active_model ?? "No model loaded"}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<div className="rounded-full border border-border px-2.5 py-1 text-xs capitalize text-muted-foreground">
|
||||
{statusLabel}
|
||||
</div>
|
||||
{/* Always rendered, disabled when idle: the only manual release must stay visible. */}
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => void unloadActiveModel()}
|
||||
disabled={unloading || !data?.active_model}
|
||||
title={
|
||||
data?.active_model
|
||||
? "Unload the model and free its VRAM"
|
||||
: "No model is loaded"
|
||||
}
|
||||
>
|
||||
<PowerOffIcon className="size-3.5" />
|
||||
{unloading ? "Unloading" : "Unload"}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => void loadMonitor()}
|
||||
disabled={refreshing}
|
||||
>
|
||||
<RefreshCwIcon
|
||||
className={cn("size-3.5", refreshing && "animate-spin")}
|
||||
/>
|
||||
Refresh
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between border-b border-border/60 px-4 py-2 text-xs text-muted-foreground">
|
||||
<span>
|
||||
{(data?.active_requests ?? 0).toLocaleString()} active /{" "}
|
||||
{entries.length.toLocaleString()} recent
|
||||
</span>
|
||||
{data?.context_length ? (
|
||||
<span>{data.context_length.toLocaleString()} context</span>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="max-h-[420px] min-h-24 overflow-y-auto p-3">
|
||||
{error ? (
|
||||
<div className="rounded-lg border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive">
|
||||
{error}
|
||||
</div>
|
||||
) : entries.length === 0 ? (
|
||||
<div className="rounded-lg border border-border/70 p-4 text-sm text-muted-foreground">
|
||||
No API traffic yet
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid gap-3">
|
||||
{visible.map((entry) =>
|
||||
isLifecycle(entry) ? (
|
||||
<LifecycleEntry key={entry.id} entry={entry} />
|
||||
) : (
|
||||
<MonitorEntry
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
detail={details[entry.id]}
|
||||
expanded={expandedIds.has(entry.id)}
|
||||
loading={loadingDetails.has(entry.id)}
|
||||
onToggle={() => toggleEntry(entry)}
|
||||
/>
|
||||
),
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Also while frozen: retention can shrink that list below one page, and hiding the
|
||||
pager would strand the console on a stale snapshot. */}
|
||||
{ordered.length > PAGE_SIZE || frozenIds !== null ? (
|
||||
<div className="flex items-center justify-between gap-2 border-t border-border/60 px-4 py-2 text-xs text-muted-foreground">
|
||||
<span>
|
||||
Page {pageIndex + 1} of {pageCount}
|
||||
{newerCount > 0 ? ` (${newerCount.toLocaleString()} new)` : ""}
|
||||
</span>
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="h-7 px-2 text-xs"
|
||||
onClick={() => goToPage(pageIndex - 1)}
|
||||
disabled={pageIndex === 0 && frozenIds === null}
|
||||
>
|
||||
Newer
|
||||
</Button>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="h-7 px-2 text-xs"
|
||||
onClick={() => goToPage(pageIndex + 1)}
|
||||
disabled={pageIndex >= pageCount - 1}
|
||||
>
|
||||
Older
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// The monitor moved onto its own page, normally reached from the floating
|
||||
// panel; this card is the way in from Settings.
|
||||
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
// Direct path, not the barrel: the barrel re-exports the page, which would pull
|
||||
// it into this chunk and defeat the route's dynamic import.
|
||||
import { useApiMonitorOverlayStore } from "@/features/api-monitor/overlay-store";
|
||||
import { getApiMonitor } from "@/features/chat/api/chat-api";
|
||||
import type { ApiMonitorResponse } from "@/features/chat/types/api";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ActivityIcon, ArrowRight02Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { useNavigate } from "@tanstack/react-router";
|
||||
import { type ReactElement, useEffect, useState } from "react";
|
||||
import { useSettingsDialogStore } from "../stores/settings-dialog-store";
|
||||
|
||||
export function MonitorLink(): ReactElement {
|
||||
const navigate = useNavigate();
|
||||
const [data, setData] = useState<ApiMonitorResponse | null>(null);
|
||||
const autoOpen = useApiMonitorOverlayStore((s) => s.autoOpen);
|
||||
const setAutoOpen = useApiMonitorOverlayStore((s) => s.setAutoOpen);
|
||||
|
||||
// One snapshot, not a poll: the live view is the monitor page.
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void getApiMonitor()
|
||||
.then((next) => {
|
||||
if (!cancelled) setData(next);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const active = data?.active_requests ?? 0;
|
||||
const recent = data?.entries.length ?? 0;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
useSettingsDialogStore.getState().closeDialog();
|
||||
void navigate({ to: "/api-monitor" });
|
||||
}}
|
||||
className="flex w-full min-w-0 items-center gap-3 rounded-lg border border-border/70 bg-background px-4 py-3 text-left transition-colors hover:bg-muted/40 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
>
|
||||
<span className="relative flex size-8 shrink-0 items-center justify-center rounded-md border border-border/70 bg-muted/40">
|
||||
<HugeiconsIcon
|
||||
icon={ActivityIcon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
{active > 0 ? (
|
||||
<span className="absolute right-1 top-1 size-2 rounded-full bg-emerald-500" />
|
||||
) : null}
|
||||
</span>
|
||||
<span className="flex min-w-0 flex-col">
|
||||
<span className="text-sm font-semibold text-foreground">
|
||||
API monitor
|
||||
</span>
|
||||
<span className="truncate text-xs text-muted-foreground">
|
||||
{data == null
|
||||
? "Live requests, errors and token usage"
|
||||
: `${active.toLocaleString()} active · ${recent.toLocaleString()} recent · ${
|
||||
data.active_model ?? "no model loaded"
|
||||
}`}
|
||||
</span>
|
||||
</span>
|
||||
<HugeiconsIcon
|
||||
icon={ArrowRight02Icon}
|
||||
strokeWidth={1.75}
|
||||
className={cn("ml-auto size-4 shrink-0 text-muted-foreground")}
|
||||
/>
|
||||
</button>
|
||||
|
||||
{/* Where the panel's own "stop opening this" gets turned back on. */}
|
||||
<div className="flex items-center justify-between gap-3 rounded-lg px-1 py-1">
|
||||
<span className="flex min-w-0 flex-col">
|
||||
<span className="text-sm text-foreground">
|
||||
Show the floating monitor automatically
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Opens a small panel when API traffic arrives.
|
||||
</span>
|
||||
</span>
|
||||
<Switch
|
||||
checked={autoOpen}
|
||||
onCheckedChange={setAutoOpen}
|
||||
aria-label="Show the floating API monitor automatically"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ import { translate, useT } from "@/i18n";
|
|||
import { AnimatePresence, motion, useReducedMotion } from "motion/react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { fetchApiKeys, revokeApiKey, type ApiKey } from "../api/api-keys";
|
||||
import { ApiMonitorConsole } from "../components/api-monitor-console";
|
||||
import { MonitorLink } from "../components/monitor-link";
|
||||
import { ApiKeyRow } from "../components/api-key-row";
|
||||
import { CreateKeyForm } from "../components/create-key-form";
|
||||
import { ModelAutoSwitchSection } from "../components/model-auto-switch-section";
|
||||
|
|
@ -168,9 +168,9 @@ export function ApiKeysTab() {
|
|||
)}
|
||||
</section>
|
||||
|
||||
<ModelAutoSwitchSection />
|
||||
<MonitorLink />
|
||||
|
||||
<ApiMonitorConsole />
|
||||
<ModelAutoSwitchSection />
|
||||
|
||||
<UsageExamples apiKey={revealed} />
|
||||
|
||||
|
|
|
|||
187
studio/frontend/tests/api-monitor-new-traffic.test.ts
Normal file
187
studio/frontend/tests/api-monitor-new-traffic.test.ts
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
// The overlay is a .tsx pulling in motion, hugeicons and the router, so it cannot
|
||||
// be imported here. Its new-traffic decision lives in a plain module for exactly
|
||||
// that reason, and this drives the real one the overlay calls.
|
||||
import {
|
||||
type WatchedEntry,
|
||||
type WatchedResponse,
|
||||
createWatch,
|
||||
observeResponse,
|
||||
rearmWatch,
|
||||
startWatching,
|
||||
} from "../src/features/api-monitor/new-traffic.ts";
|
||||
|
||||
// The server's clock. Entry timestamps are its time.time(), so the tests keep them
|
||||
// in those units and never mix in a browser instant.
|
||||
const SERVER_NOW = 1_000_000;
|
||||
// performance.now() when the poll stood up.
|
||||
const WATCH_AT = 1_000;
|
||||
|
||||
function entry(
|
||||
id: string,
|
||||
status: WatchedEntry["status"],
|
||||
startedAt: number,
|
||||
viaApiKey = true,
|
||||
): WatchedEntry {
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
return { id, status, via_api_key: viaApiKey, started_at: startedAt };
|
||||
}
|
||||
|
||||
function snapshot(
|
||||
entries: WatchedEntry[],
|
||||
serverTime: number | null = SERVER_NOW,
|
||||
): WatchedResponse {
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
return { entries, server_time: serverTime };
|
||||
}
|
||||
|
||||
function watchFrom(startedAtMs: number) {
|
||||
const watch = createWatch(0);
|
||||
startWatching(watch, startedAtMs);
|
||||
return watch;
|
||||
}
|
||||
|
||||
test("a call that finished before the first snapshot arrived is new traffic", () => {
|
||||
// The tab was hidden for 4s after the poll stood up, so poll() issued no fetch.
|
||||
// The user's first curl ran 2s into that gap and was already done when the
|
||||
// snapshot finally landed. Terminal, but not history.
|
||||
const opened = observeResponse(
|
||||
watchFrom(WATCH_AT),
|
||||
snapshot([entry("apireq_new", "completed", SERVER_NOW - 2)]),
|
||||
WATCH_AT + 4_000,
|
||||
);
|
||||
assert.equal(opened, true);
|
||||
});
|
||||
|
||||
test("traffic from before the watch began stays history", () => {
|
||||
const opened = observeResponse(
|
||||
watchFrom(WATCH_AT),
|
||||
snapshot([entry("apireq_old", "completed", SERVER_NOW - 90)]),
|
||||
WATCH_AT + 4_000,
|
||||
);
|
||||
assert.equal(opened, false);
|
||||
});
|
||||
|
||||
test("a request still running at the first snapshot is live traffic", () => {
|
||||
const opened = observeResponse(
|
||||
watchFrom(WATCH_AT),
|
||||
snapshot([entry("apireq_live", "running", SERVER_NOW - 90)]),
|
||||
WATCH_AT + 10,
|
||||
);
|
||||
assert.equal(opened, true);
|
||||
});
|
||||
|
||||
test("a fresh id in a later snapshot opens the panel", () => {
|
||||
const watch = watchFrom(WATCH_AT);
|
||||
const backlog = [entry("apireq_old", "completed", SERVER_NOW - 90)];
|
||||
assert.equal(observeResponse(watch, snapshot(backlog), WATCH_AT + 10), false);
|
||||
const opened = observeResponse(
|
||||
watch,
|
||||
snapshot(
|
||||
[entry("apireq_next", "completed", SERVER_NOW + 4), ...backlog],
|
||||
SERVER_NOW + 5,
|
||||
),
|
||||
WATCH_AT + 5_010,
|
||||
);
|
||||
assert.equal(opened, true);
|
||||
});
|
||||
|
||||
test("Studio's own chat never opens the panel", () => {
|
||||
const opened = observeResponse(
|
||||
watchFrom(WATCH_AT),
|
||||
snapshot([entry("uireq", "completed", SERVER_NOW - 2, false)]),
|
||||
WATCH_AT + 4_000,
|
||||
);
|
||||
assert.equal(opened, false);
|
||||
});
|
||||
|
||||
test("a backend with no clock field keeps the old terminal-is-history seed", () => {
|
||||
const opened = observeResponse(
|
||||
watchFrom(WATCH_AT),
|
||||
snapshot([entry("apireq_new", "completed", SERVER_NOW - 2)], null),
|
||||
WATCH_AT + 4_000,
|
||||
);
|
||||
assert.equal(opened, false);
|
||||
});
|
||||
|
||||
test("a browser clock disagreeing with the server's does not replay the backlog", () => {
|
||||
// The cutoff is the server's own clock minus a browser DURATION, never minus a
|
||||
// browser timestamp, so a browser whose wall clock is minutes off still dates
|
||||
// the backlog correctly.
|
||||
const opened = observeResponse(
|
||||
watchFrom(WATCH_AT),
|
||||
snapshot([
|
||||
entry("apireq_a", "completed", SERVER_NOW - 300),
|
||||
entry("apireq_b", "completed", SERVER_NOW - 120),
|
||||
]),
|
||||
WATCH_AT + 20,
|
||||
);
|
||||
assert.equal(opened, false);
|
||||
});
|
||||
|
||||
test("coming back from the full page does not replay the rows it showed", () => {
|
||||
const watch = watchFrom(WATCH_AT);
|
||||
const backlog = [entry("apireq_old", "completed", SERVER_NOW - 90)];
|
||||
observeResponse(watch, snapshot(backlog), WATCH_AT + 10);
|
||||
// 60s on /api-monitor reading those rows, then back to chat.
|
||||
rearmWatch(watch);
|
||||
startWatching(watch, WATCH_AT + 60_000);
|
||||
const opened = observeResponse(
|
||||
watch,
|
||||
snapshot(backlog, SERVER_NOW + 60),
|
||||
WATCH_AT + 60_010,
|
||||
);
|
||||
assert.equal(opened, false);
|
||||
});
|
||||
|
||||
test("a request still running when the full page is left does not reopen the overlay", () => {
|
||||
// The user opened /api-monitor to watch a long generation, then went back to
|
||||
// chat while it was still running. That row was on screen the whole time.
|
||||
const watch = watchFrom(WATCH_AT);
|
||||
const live = entry("apireq_live", "running", SERVER_NOW - 5);
|
||||
observeResponse(watch, snapshot([live]), WATCH_AT + 10);
|
||||
rearmWatch(watch);
|
||||
startWatching(watch, WATCH_AT + 60_000);
|
||||
const opened = observeResponse(
|
||||
watch,
|
||||
snapshot([live], SERVER_NOW + 60),
|
||||
WATCH_AT + 60_010,
|
||||
);
|
||||
assert.equal(opened, false);
|
||||
});
|
||||
|
||||
test("a rearm writes off only the snapshot it comes back to", () => {
|
||||
// The write-off is one seed, not a mode: a call that arrives after the return
|
||||
// is still new traffic.
|
||||
const watch = watchFrom(WATCH_AT);
|
||||
const live = entry("apireq_live", "running", SERVER_NOW - 5);
|
||||
observeResponse(watch, snapshot([live]), WATCH_AT + 10);
|
||||
rearmWatch(watch);
|
||||
startWatching(watch, WATCH_AT + 60_000);
|
||||
observeResponse(watch, snapshot([live], SERVER_NOW + 60), WATCH_AT + 60_010);
|
||||
const opened = observeResponse(
|
||||
watch,
|
||||
snapshot(
|
||||
[entry("apireq_next", "running", SERVER_NOW + 61), live],
|
||||
SERVER_NOW + 62,
|
||||
),
|
||||
WATCH_AT + 62_010,
|
||||
);
|
||||
assert.equal(opened, true);
|
||||
});
|
||||
|
||||
test("a fresh watch still reports a request that was already running", () => {
|
||||
// The rearm write-off must not become the default seed for a session that
|
||||
// never saw the full page.
|
||||
const opened = observeResponse(
|
||||
watchFrom(WATCH_AT),
|
||||
snapshot([entry("apireq_live", "running", SERVER_NOW - 90)]),
|
||||
WATCH_AT + 10,
|
||||
);
|
||||
assert.equal(opened, true);
|
||||
});
|
||||
36
studio/frontend/tests/bundler-resolver.mjs
Normal file
36
studio/frontend/tests/bundler-resolver.mjs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// The two resolution rules vite and tsconfig's "bundler" mode give the app that
|
||||
// bare node does not have: the "@/*" path alias, and a relative import written
|
||||
// without its extension. Register this from a test that needs to import a src
|
||||
// module using either, which is otherwise unreachable from the test runner.
|
||||
import { existsSync } from "node:fs";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const SRC = fileURLToPath(new URL("../src/", import.meta.url));
|
||||
|
||||
function firstExisting(base) {
|
||||
for (const candidate of [`${base}.ts`, `${base}/index.ts`, base]) {
|
||||
if (existsSync(candidate)) {
|
||||
return pathToFileURL(candidate).href;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolve(specifier, context, next) {
|
||||
if (specifier.startsWith("@/")) {
|
||||
const resolved = firstExisting(SRC + specifier.slice(2));
|
||||
return next(resolved ?? specifier, context);
|
||||
}
|
||||
if (specifier.startsWith(".") && context.parentURL?.startsWith("file:")) {
|
||||
const resolved = firstExisting(
|
||||
fileURLToPath(new URL(specifier, context.parentURL)),
|
||||
);
|
||||
if (resolved) {
|
||||
return next(resolved, context);
|
||||
}
|
||||
}
|
||||
return next(specifier, context);
|
||||
}
|
||||
131
studio/frontend/tests/helpers/kit.ts
Normal file
131
studio/frontend/tests/helpers/kit.ts
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { register } from "node:module";
|
||||
|
||||
import type { ResidentAdoptionState } from "../../src/features/hub/lib/adopt-inference-status.ts";
|
||||
import type { ResidentStatusRefreshTargets } from "../../src/features/hub/lib/resident-status-refresh.ts";
|
||||
|
||||
/**
|
||||
* Teach the loader the two resolution rules vite and tsconfig's "bundler" mode
|
||||
* give the app. Call this before the dynamic import of any src module that
|
||||
* resolves the way vite and tsconfig resolve, not the way bare node does.
|
||||
*/
|
||||
export function registerBundlerResolver(): void {
|
||||
register("../bundler-resolver.mjs", import.meta.url);
|
||||
}
|
||||
|
||||
export type StorageFake = {
|
||||
getItem: (key: string) => string | null;
|
||||
setItem: (key: string, value: string) => void;
|
||||
removeItem: (key: string) => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* An in-memory localStorage, installed on globalThis under both the names the
|
||||
* app reads it by. The returned map is the backing store, so a test can stage
|
||||
* records before the module under test is imported.
|
||||
*/
|
||||
export function installLocalStorageFake(): {
|
||||
store: Map<string, string>;
|
||||
storage: StorageFake;
|
||||
} {
|
||||
const store = new Map<string, string>();
|
||||
const storage: StorageFake = {
|
||||
getItem: (key: string) => store.get(key) ?? null,
|
||||
setItem: (key: string, value: string) => {
|
||||
store.set(key, value);
|
||||
},
|
||||
removeItem: (key: string) => {
|
||||
store.delete(key);
|
||||
},
|
||||
};
|
||||
Object.assign(globalThis, {
|
||||
window: { localStorage: storage },
|
||||
localStorage: storage,
|
||||
});
|
||||
return { store, storage };
|
||||
}
|
||||
|
||||
/** The chat-runtime store as it stands before anything has hydrated it. */
|
||||
export function emptyStore(
|
||||
overrides: Partial<ResidentAdoptionState> = {},
|
||||
): ResidentAdoptionState {
|
||||
return {
|
||||
checkpoint: null,
|
||||
checkpointIsExternal: false,
|
||||
activeGgufVariant: null,
|
||||
modelLoading: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
/** Records the store actions adoptResidentModelStatus takes, in order. */
|
||||
export function spies() {
|
||||
const calls: string[] = [];
|
||||
const previouslySeen: { checkpoint: string | null; ggufVariant: string | null }[] =
|
||||
[];
|
||||
return {
|
||||
calls,
|
||||
previouslySeen,
|
||||
actions: {
|
||||
setCheckpoint(checkpointId: string, ggufVariant: string | null) {
|
||||
calls.push(`setCheckpoint:${checkpointId}:${ggufVariant ?? ""}`);
|
||||
},
|
||||
applyStatus(previous: {
|
||||
checkpoint: string | null;
|
||||
ggufVariant: string | null;
|
||||
}) {
|
||||
calls.push("applyStatus");
|
||||
previouslySeen.push(previous);
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** A window/document pair whose events and visibility a test drives by hand. */
|
||||
export function fakeTargets(): ResidentStatusRefreshTargets & {
|
||||
hidden: boolean;
|
||||
fire: (target: "window" | "document", type: string) => void;
|
||||
listenerCount: () => number;
|
||||
} {
|
||||
const listeners = new Map<string, Set<EventListenerOrEventListenerObject>>();
|
||||
const key = (target: string, type: string) => `${target}:${type}`;
|
||||
const make = (target: "window" | "document") => ({
|
||||
addEventListener(type: string, fn: EventListenerOrEventListenerObject) {
|
||||
const set = listeners.get(key(target, type)) ?? new Set();
|
||||
set.add(fn);
|
||||
listeners.set(key(target, type), set);
|
||||
},
|
||||
removeEventListener(type: string, fn: EventListenerOrEventListenerObject) {
|
||||
listeners.get(key(target, type))?.delete(fn);
|
||||
},
|
||||
});
|
||||
const visibility = { hidden: false };
|
||||
const state = {
|
||||
get hidden() {
|
||||
return visibility.hidden;
|
||||
},
|
||||
set hidden(next: boolean) {
|
||||
visibility.hidden = next;
|
||||
},
|
||||
window: make("window"),
|
||||
document: {
|
||||
...make("document"),
|
||||
get hidden() {
|
||||
return visibility.hidden;
|
||||
},
|
||||
},
|
||||
fire(target: "window" | "document", type: string) {
|
||||
for (const fn of listeners.get(key(target, type)) ?? []) {
|
||||
(fn as EventListener)(new Event(type));
|
||||
}
|
||||
},
|
||||
listenerCount() {
|
||||
let total = 0;
|
||||
for (const set of listeners.values()) total += set.size;
|
||||
return total;
|
||||
},
|
||||
};
|
||||
return state as never;
|
||||
}
|
||||
311
studio/frontend/tests/hub-resident-status.test.ts
Normal file
311
studio/frontend/tests/hub-resident-status.test.ts
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { adoptResidentModelStatus } from "../src/features/hub/lib/adopt-inference-status.ts";
|
||||
import {
|
||||
ggufVariantsMatch,
|
||||
residentModelIdMatches,
|
||||
} from "../src/features/hub/lib/model-identity.ts";
|
||||
import { subscribeResidentStatusRefresh } from "../src/features/hub/lib/resident-status-refresh.ts";
|
||||
import { emptyStore, fakeTargets, spies } from "./helpers/kit.ts";
|
||||
|
||||
const RESIDENT = {
|
||||
checkpointId: "unsloth/Qwen3-8B-GGUF",
|
||||
ggufVariant: "Q4_K_M",
|
||||
};
|
||||
|
||||
/**
|
||||
* Store actions that refuse to be called. The message on each names what must
|
||||
* not happen, so a test states its rule by the action it declines to forbid.
|
||||
*/
|
||||
function refusing(messages: {
|
||||
setCheckpoint?: string;
|
||||
clearCheckpoint?: string;
|
||||
applyStatus?: string;
|
||||
}) {
|
||||
const refuse = (message = "unreachable") => {
|
||||
return () => {
|
||||
throw new Error(message);
|
||||
};
|
||||
};
|
||||
return {
|
||||
setCheckpoint: refuse(messages.setCheckpoint),
|
||||
clearCheckpoint: refuse(messages.clearCheckpoint),
|
||||
applyStatus: refuse(messages.applyStatus),
|
||||
};
|
||||
}
|
||||
|
||||
test("landing on the Hub applies the whole status, not just the checkpoint", () => {
|
||||
// Nothing else on /hub hydrates the runtime store: useChatModelRuntime has no
|
||||
// mount sync and the chat page is a different route. Pinning only the
|
||||
// checkpoint leaves every field useActiveModelConfig reads at its default, so
|
||||
// the settings page offers those defaults as the resident model's live config.
|
||||
const { calls, actions } = spies();
|
||||
const adopted = adoptResidentModelStatus(RESIDENT, emptyStore(), actions);
|
||||
assert.equal(adopted, true);
|
||||
assert.deepEqual(calls, [
|
||||
"setCheckpoint:unsloth/Qwen3-8B-GGUF:Q4_K_M",
|
||||
"applyStatus",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a checkpoint that already matches is still hydrated", () => {
|
||||
// A reload rehydrates params.checkpoint from localStorage on its own, with
|
||||
// none of the fields that say how the model was actually launched.
|
||||
const { calls, actions } = spies();
|
||||
adoptResidentModelStatus(
|
||||
RESIDENT,
|
||||
emptyStore({
|
||||
checkpoint: "unsloth/Qwen3-8B-GGUF",
|
||||
activeGgufVariant: "Q4_K_M",
|
||||
}),
|
||||
actions,
|
||||
);
|
||||
assert.deepEqual(calls, ["applyStatus"]);
|
||||
});
|
||||
|
||||
test("an API auto-switch under the tab re-pins the model and the quant", () => {
|
||||
for (const stale of [
|
||||
{ checkpoint: "unsloth/Llama-3.1-8B-GGUF", activeGgufVariant: "Q4_K_M" },
|
||||
{ checkpoint: "unsloth/Qwen3-8B-GGUF", activeGgufVariant: "Q8_0" },
|
||||
]) {
|
||||
const { calls, actions } = spies();
|
||||
adoptResidentModelStatus(RESIDENT, emptyStore(stale), actions);
|
||||
assert.deepEqual(calls, [
|
||||
"setCheckpoint:unsloth/Qwen3-8B-GGUF:Q4_K_M",
|
||||
"applyStatus",
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
test("the status applied is the one from before the checkpoint moved", () => {
|
||||
// applyActiveModelStatusToStore tells a hydration from steady state by the
|
||||
// previous checkpoint/quant, so it has to be read before setCheckpoint syncs
|
||||
// them, or a variant-only switch reads as steady state and keeps the old
|
||||
// quant's baselines.
|
||||
const { previouslySeen, actions } = spies();
|
||||
adoptResidentModelStatus(
|
||||
RESIDENT,
|
||||
emptyStore({
|
||||
checkpoint: "unsloth/Qwen3-8B-GGUF",
|
||||
activeGgufVariant: "Q8_0",
|
||||
}),
|
||||
actions,
|
||||
);
|
||||
assert.deepEqual(previouslySeen, [
|
||||
{ checkpoint: "unsloth/Qwen3-8B-GGUF", ggufVariant: "Q8_0" },
|
||||
]);
|
||||
});
|
||||
|
||||
test("nothing is adopted when no model is loaded", () => {
|
||||
const { calls, actions } = spies();
|
||||
const adopted = adoptResidentModelStatus(
|
||||
{ checkpointId: null, ggufVariant: null },
|
||||
emptyStore(),
|
||||
actions,
|
||||
);
|
||||
assert.equal(adopted, false);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test("an external-provider selection is left alone", () => {
|
||||
// It has no local mirror, so stamping the resident GGUF's launch settings onto
|
||||
// it would describe a model the user is not talking to.
|
||||
const { calls, actions } = spies();
|
||||
const adopted = adoptResidentModelStatus(
|
||||
RESIDENT,
|
||||
emptyStore({
|
||||
checkpoint: "openai/gpt-5",
|
||||
checkpointIsExternal: true,
|
||||
}),
|
||||
actions,
|
||||
);
|
||||
assert.equal(adopted, false);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test("a load in flight is not fought", () => {
|
||||
// The load applies its own status when it settles, and the load dialog owns
|
||||
// the params meanwhile.
|
||||
const { calls, actions } = spies();
|
||||
const adopted = adoptResidentModelStatus(
|
||||
RESIDENT,
|
||||
emptyStore({ modelLoading: true }),
|
||||
actions,
|
||||
);
|
||||
assert.equal(adopted, false);
|
||||
assert.deepEqual(calls, []);
|
||||
});
|
||||
|
||||
test("an empty status drops a local checkpoint the server no longer has", () => {
|
||||
// Unloading from another tab, from the API monitor or over the API leaves this
|
||||
// store pinned; the settings page then treats the row as resident and seeds the
|
||||
// editor from a launch config nothing is running.
|
||||
const cleared: string[] = [];
|
||||
const adopted = adoptResidentModelStatus(
|
||||
{ checkpointId: null, ggufVariant: null },
|
||||
emptyStore({
|
||||
checkpoint: "/models/llama.gguf",
|
||||
activeGgufVariant: "Q4_K_M",
|
||||
}),
|
||||
{
|
||||
...refusing({
|
||||
setCheckpoint: "nothing is resident, so nothing may be pinned",
|
||||
applyStatus: "there is no status to apply",
|
||||
}),
|
||||
clearCheckpoint: () => {
|
||||
cleared.push("cleared");
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(adopted, true);
|
||||
assert.deepEqual(cleared, ["cleared"]);
|
||||
});
|
||||
|
||||
test("an empty status leaves an external pick alone", () => {
|
||||
// clearCheckpoint also drops the persisted external selection, so an empty
|
||||
// status must not reach it: the local model is not what the user is talking to.
|
||||
const adopted = adoptResidentModelStatus(
|
||||
{ checkpointId: null, ggufVariant: null },
|
||||
emptyStore({
|
||||
checkpoint: "gemini/gemini-2.5-pro",
|
||||
checkpointIsExternal: true,
|
||||
}),
|
||||
refusing({
|
||||
clearCheckpoint: "an external pick must survive an empty status",
|
||||
}),
|
||||
);
|
||||
assert.equal(adopted, false);
|
||||
});
|
||||
|
||||
test("an empty status does not fight a load this tab started", () => {
|
||||
const adopted = adoptResidentModelStatus(
|
||||
{ checkpointId: null, ggufVariant: null },
|
||||
emptyStore({
|
||||
checkpoint: "/models/llama.gguf",
|
||||
modelLoading: true,
|
||||
}),
|
||||
refusing({ clearCheckpoint: "the load owns the store until it settles" }),
|
||||
);
|
||||
assert.equal(adopted, false);
|
||||
});
|
||||
|
||||
test("an empty status on an already empty store changes nothing", () => {
|
||||
const adopted = adoptResidentModelStatus(
|
||||
{ checkpointId: null, ggufVariant: null },
|
||||
emptyStore(),
|
||||
refusing({ clearCheckpoint: "there is nothing to clear" }),
|
||||
);
|
||||
assert.equal(adopted, false);
|
||||
});
|
||||
|
||||
test("coming back to the window re-reads inference status", () => {
|
||||
// An OpenAI-compatible request auto-switches the resident model whenever it
|
||||
// likes. The Hub's only other status read is its mount effect, so without this
|
||||
// the catalog and the settings page keep describing the previous model for as
|
||||
// long as the Hub stays mounted.
|
||||
const targets = fakeTargets();
|
||||
let reads = 0;
|
||||
subscribeResidentStatusRefresh(() => {
|
||||
reads += 1;
|
||||
}, targets);
|
||||
|
||||
assert.equal(reads, 0, "subscribing must not read on its own");
|
||||
targets.fire("window", "focus");
|
||||
assert.equal(reads, 1);
|
||||
targets.fire("document", "visibilitychange");
|
||||
assert.equal(reads, 2);
|
||||
});
|
||||
|
||||
test("a tab going hidden does not read", () => {
|
||||
// visibilitychange fires on the way out too, and a hidden tab has no settings
|
||||
// page to correct.
|
||||
const targets = fakeTargets();
|
||||
let reads = 0;
|
||||
subscribeResidentStatusRefresh(() => {
|
||||
reads += 1;
|
||||
}, targets);
|
||||
|
||||
targets.hidden = true;
|
||||
targets.fire("document", "visibilitychange");
|
||||
assert.equal(reads, 0);
|
||||
|
||||
targets.hidden = false;
|
||||
targets.fire("document", "visibilitychange");
|
||||
assert.equal(reads, 1);
|
||||
});
|
||||
|
||||
test("an auto-switch under a mounted Hub stops hiding the live config", () => {
|
||||
// The whole point, end to end: while the Hub is mounted an OpenAI-compatible
|
||||
// request swaps the resident model. Without a second read the store still names
|
||||
// the old one, so hub-page's settingsTargetIsResident says the newly loaded
|
||||
// model is not resident, its settings page is handed loadedConfig=null, and
|
||||
// ModelConfigPage seeds the editor from saved/default values -- which Apply then
|
||||
// reloads the model with, over what the API actually selected.
|
||||
const store = emptyStore({
|
||||
checkpoint: "unsloth/Qwen3-8B-GGUF",
|
||||
activeGgufVariant: "Q4_K_M",
|
||||
});
|
||||
// What the server reports once the API request has switched it.
|
||||
let serverStatus = {
|
||||
checkpointId: "unsloth/Llama-3.1-8B-Instruct-GGUF",
|
||||
ggufVariant: "Q8_0",
|
||||
};
|
||||
const readStatusAndAdopt = () => {
|
||||
adoptResidentModelStatus(
|
||||
serverStatus,
|
||||
{ ...store },
|
||||
{
|
||||
setCheckpoint: (checkpointId, ggufVariant) => {
|
||||
store.checkpoint = checkpointId;
|
||||
store.activeGgufVariant = ggufVariant;
|
||||
},
|
||||
applyStatus: () => undefined,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
// hub-page.tsx's settingsTargetIsResident, for the model the API just loaded.
|
||||
const settingsTargetIsResident = () =>
|
||||
residentModelIdMatches(store.checkpoint, serverStatus.checkpointId) &&
|
||||
ggufVariantsMatch(store.activeGgufVariant, serverStatus.ggufVariant);
|
||||
|
||||
const targets = fakeTargets();
|
||||
subscribeResidentStatusRefresh(readStatusAndAdopt, targets);
|
||||
|
||||
assert.equal(
|
||||
settingsTargetIsResident(),
|
||||
false,
|
||||
"precondition: the mount-time read predates the switch",
|
||||
);
|
||||
targets.fire("window", "focus");
|
||||
assert.equal(settingsTargetIsResident(), true);
|
||||
|
||||
// A load this tab started owns the store until it settles, so a refresh landing
|
||||
// mid-switch must not re-pin the model the user is moving away from.
|
||||
store.modelLoading = true;
|
||||
serverStatus = {
|
||||
checkpointId: "unsloth/Qwen3-8B-GGUF",
|
||||
ggufVariant: "Q4_K_M",
|
||||
};
|
||||
targets.fire("window", "focus");
|
||||
assert.equal(store.checkpoint, "unsloth/Llama-3.1-8B-Instruct-GGUF");
|
||||
});
|
||||
|
||||
test("unsubscribing stops the reads and leaves no listener behind", () => {
|
||||
const targets = fakeTargets();
|
||||
let reads = 0;
|
||||
const unsubscribe = subscribeResidentStatusRefresh(() => {
|
||||
reads += 1;
|
||||
}, targets);
|
||||
|
||||
assert.equal(targets.listenerCount(), 2);
|
||||
unsubscribe();
|
||||
assert.equal(targets.listenerCount(), 0);
|
||||
targets.fire("window", "focus");
|
||||
targets.fire("document", "visibilitychange");
|
||||
assert.equal(reads, 0);
|
||||
});
|
||||
132
studio/frontend/tests/model-config-instance-key.test.ts
Normal file
132
studio/frontend/tests/model-config-instance-key.test.ts
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { modelConfigInstanceKey } from "../src/features/model-picker/model-config/config-signature.ts";
|
||||
import type { PerModelConfig } from "../src/features/model-picker/model-config/per-model-config.ts";
|
||||
|
||||
const MODEL = "unsloth/Qwen3-8B-GGUF";
|
||||
const VARIANT = "Q4_K_M";
|
||||
|
||||
// What the model is actually running with, as useActiveModelConfig reports it.
|
||||
const LIVE: PerModelConfig = {
|
||||
customContextLength: 16384,
|
||||
maxSeqLength: null,
|
||||
kvCacheDtype: "q8_0",
|
||||
speculativeType: "ngram",
|
||||
specDraftNMax: 6,
|
||||
nParallel: 4,
|
||||
tensorParallel: true,
|
||||
chatTemplateOverride: null,
|
||||
gpuMemoryMode: "manual",
|
||||
gpuLayers: 24,
|
||||
nCpuMoe: 3,
|
||||
selectedGpuIds: [0, 1],
|
||||
};
|
||||
|
||||
// What ModelConfigPage would fall back to before the live config lands.
|
||||
const SAVED: PerModelConfig = {
|
||||
customContextLength: null,
|
||||
maxSeqLength: null,
|
||||
kvCacheDtype: null,
|
||||
speculativeType: "auto",
|
||||
specDraftNMax: null,
|
||||
nParallel: null,
|
||||
tensorParallel: false,
|
||||
chatTemplateOverride: null,
|
||||
gpuMemoryMode: "auto",
|
||||
gpuLayers: -1,
|
||||
nCpuMoe: 0,
|
||||
selectedGpuIds: null,
|
||||
};
|
||||
|
||||
/**
|
||||
* ModelConfigPage reads `loadedConfig` in a useState initializer, so it seeds its
|
||||
* editable state once per MOUNTED instance; React keeps that instance for as long
|
||||
* as the key is unchanged. This is that rule, and nothing else.
|
||||
*/
|
||||
function renderEditor(
|
||||
previous: { key: string; editing: PerModelConfig } | null,
|
||||
key: string,
|
||||
loadedConfig: PerModelConfig | null,
|
||||
): { key: string; editing: PerModelConfig } {
|
||||
if (previous && previous.key === key) {
|
||||
return previous;
|
||||
}
|
||||
return { key, editing: loadedConfig ?? SAVED };
|
||||
}
|
||||
|
||||
test("the settings editor re-seeds when the live config arrives after mount", () => {
|
||||
// Opened before /api/inference/status answered, or while the target was still
|
||||
// loading: loadedConfig is null on the first render and live on the next.
|
||||
let editor = renderEditor(
|
||||
null,
|
||||
modelConfigInstanceKey(MODEL, VARIANT, null),
|
||||
null,
|
||||
);
|
||||
assert.deepEqual(editor.editing, SAVED);
|
||||
|
||||
editor = renderEditor(
|
||||
editor,
|
||||
modelConfigInstanceKey(MODEL, VARIANT, LIVE),
|
||||
LIVE,
|
||||
);
|
||||
// Without the live config in the key the editor would still hold SAVED, and
|
||||
// Apply would reload the model with it over what it is running with.
|
||||
assert.deepEqual(editor.editing, LIVE);
|
||||
});
|
||||
|
||||
test("a repeated status poll keeps the same editor instance", () => {
|
||||
const first = renderEditor(
|
||||
null,
|
||||
modelConfigInstanceKey(MODEL, VARIANT, LIVE),
|
||||
LIVE,
|
||||
);
|
||||
// A structurally equal config from the next poll must not remount and throw
|
||||
// away whatever the user has typed since.
|
||||
const again = renderEditor(
|
||||
first,
|
||||
modelConfigInstanceKey(MODEL, VARIANT, { ...LIVE }),
|
||||
LIVE,
|
||||
);
|
||||
assert.equal(again, first);
|
||||
});
|
||||
|
||||
test("every mirrored setting moves the instance key", () => {
|
||||
const base = modelConfigInstanceKey(MODEL, VARIANT, LIVE);
|
||||
const changes: PerModelConfig[] = [
|
||||
{ ...LIVE, customContextLength: 8192 },
|
||||
{ ...LIVE, maxSeqLength: 4096 },
|
||||
{ ...LIVE, kvCacheDtype: "f16" },
|
||||
{ ...LIVE, speculativeType: "off" },
|
||||
{ ...LIVE, specDraftNMax: 4 },
|
||||
{ ...LIVE, nParallel: 1 },
|
||||
{ ...LIVE, tensorParallel: false },
|
||||
{ ...LIVE, chatTemplateOverride: "{{ bos_token }}" },
|
||||
{ ...LIVE, gpuMemoryMode: "auto" },
|
||||
{ ...LIVE, gpuLayers: 20 },
|
||||
{ ...LIVE, nCpuMoe: 0 },
|
||||
{ ...LIVE, selectedGpuIds: [0] },
|
||||
];
|
||||
for (const changed of changes) {
|
||||
assert.notEqual(modelConfigInstanceKey(MODEL, VARIANT, changed), base);
|
||||
}
|
||||
// The GPU pick is a set, not an order.
|
||||
assert.equal(
|
||||
modelConfigInstanceKey(MODEL, VARIANT, { ...LIVE, selectedGpuIds: [1, 0] }),
|
||||
base,
|
||||
);
|
||||
});
|
||||
|
||||
test("the model and its quant still key the editor", () => {
|
||||
const base = modelConfigInstanceKey(MODEL, VARIANT, LIVE);
|
||||
assert.notEqual(modelConfigInstanceKey("unsloth/Other-GGUF", VARIANT, LIVE), base);
|
||||
assert.notEqual(modelConfigInstanceKey(MODEL, "Q8_0", LIVE), base);
|
||||
// A loose .gguf carries no quant; null and undefined are the same absence.
|
||||
assert.equal(
|
||||
modelConfigInstanceKey(MODEL, null, LIVE),
|
||||
modelConfigInstanceKey(MODEL, undefined, LIVE),
|
||||
);
|
||||
});
|
||||
344
studio/frontend/tests/model-identity.test.ts
Normal file
344
studio/frontend/tests/model-identity.test.ts
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { settingsGgufVariantForRow } from "../src/features/hub/inventory/settings-identity.ts";
|
||||
import type {
|
||||
CachedInventoryRow,
|
||||
LocalInventoryRow,
|
||||
} from "../src/features/hub/inventory/types.ts";
|
||||
import {
|
||||
isOllamaLinkPath,
|
||||
modelIdsMatch,
|
||||
publicModelId,
|
||||
residentModelIdMatches,
|
||||
} from "../src/features/hub/lib/model-identity.ts";
|
||||
import {
|
||||
installLocalStorageFake,
|
||||
registerBundlerResolver,
|
||||
} from "./helpers/kit.ts";
|
||||
|
||||
registerBundlerResolver();
|
||||
const { store, storage } = installLocalStorageFake();
|
||||
|
||||
const REPO_KEY = 'v2:["unsloth/repo-gguf","q4_k_m"]';
|
||||
|
||||
// The legacy import of unsloth_load_settings runs once, on the first read after
|
||||
// load, so it has to be staged before the module is imported.
|
||||
store.set(
|
||||
"unsloth_model_configs",
|
||||
JSON.stringify({ [REPO_KEY]: { version: 1, maxSeqLength: 32768 } }),
|
||||
);
|
||||
store.set(
|
||||
"unsloth_load_settings",
|
||||
JSON.stringify({ "Unsloth/Repo-GGUF::Q4_K_M": { contextLength: 8192 } }),
|
||||
);
|
||||
|
||||
const { listPerModelConfigs, resolveInitialConfig, savePerModelConfig } =
|
||||
await import("../src/features/model-picker/model-config/per-model-config.ts");
|
||||
const { modelStorageKey, splitQuantSuffix } = await import(
|
||||
"../src/features/model-picker/model-config/model-identity.ts"
|
||||
);
|
||||
|
||||
function config(maxSeqLength: number, kvCacheDtype: string | null = null) {
|
||||
return {
|
||||
customContextLength: null,
|
||||
maxSeqLength,
|
||||
kvCacheDtype,
|
||||
speculativeType: null,
|
||||
specDraftNMax: null,
|
||||
nParallel: null,
|
||||
tensorParallel: false,
|
||||
chatTemplateOverride: null,
|
||||
};
|
||||
}
|
||||
|
||||
function storedKeys(): string[] {
|
||||
return Object.keys(
|
||||
JSON.parse(storage.getItem("unsloth_model_configs") ?? "{}"),
|
||||
);
|
||||
}
|
||||
|
||||
test("publicModelId mirrors what /status reports for a path-loaded model", () => {
|
||||
// Mirrors public_model_id in studio/backend/core/inference/model_ids.py.
|
||||
assert.equal(
|
||||
publicModelId("/srv/models/Qwen3-8B-Q4_K_M.gguf"),
|
||||
"Qwen3-8B-Q4_K_M",
|
||||
);
|
||||
assert.equal(
|
||||
publicModelId(
|
||||
"/home/u/.cache/huggingface/hub/models--unsloth--Qwen3-8B-GGUF/snapshots/abc123",
|
||||
),
|
||||
"unsloth/Qwen3-8B-GGUF",
|
||||
);
|
||||
assert.equal(publicModelId("C:\\models\\Foo-Q4_K_M.gguf"), "Foo-Q4_K_M");
|
||||
assert.equal(publicModelId("~/models/Foo.gguf"), "Foo");
|
||||
assert.equal(publicModelId("/srv/models/repo/"), "repo");
|
||||
// A repo id and an already-clean name come back untouched.
|
||||
assert.equal(publicModelId("unsloth/Qwen3-8B-GGUF"), "unsloth/Qwen3-8B-GGUF");
|
||||
assert.equal(publicModelId("Qwen3-8B-Q4_K_M"), "Qwen3-8B-Q4_K_M");
|
||||
// "models--" alone is not the cache layout; only the snapshots sibling is.
|
||||
assert.equal(publicModelId("models--only--nosnapshots/blobs/x"), "x");
|
||||
});
|
||||
|
||||
test("a resident path-loaded model is matched by the id /status reports", () => {
|
||||
// A loose .gguf: the catalog row is keyed by the path, and the Hub page records
|
||||
// the loadable identifier (status.model_identifier), so the literal pass answers.
|
||||
assert.equal(
|
||||
modelIdsMatch("Qwen3-8B-Q4_K_M", "/srv/models/Qwen3-8B-Q4_K_M.gguf"),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
residentModelIdMatches(
|
||||
"/srv/models/Qwen3-8B-Q4_K_M.gguf",
|
||||
"/srv/models/Qwen3-8B-Q4_K_M.gguf",
|
||||
"/srv/models/Qwen3-8B-Q4_K_M.gguf",
|
||||
),
|
||||
true,
|
||||
);
|
||||
// A repo in an inactive HF cache loads by snapshot path but keeps the repo id
|
||||
// as its settings identity, so the configId alias already covers it.
|
||||
assert.equal(
|
||||
residentModelIdMatches(
|
||||
"unsloth/Qwen3-8B-GGUF",
|
||||
"/mnt/old-cache/models--unsloth--Qwen3-8B-GGUF/snapshots/abc123",
|
||||
"unsloth/Qwen3-8B-GGUF",
|
||||
),
|
||||
true,
|
||||
);
|
||||
// The raw identifier is still matched literally.
|
||||
assert.equal(
|
||||
residentModelIdMatches(
|
||||
"/srv/models/Qwen3-8B-Q4_K_M.gguf",
|
||||
"/srv/models/Qwen3-8B-Q4_K_M.gguf",
|
||||
null,
|
||||
),
|
||||
true,
|
||||
);
|
||||
// Another model is still not the loaded one.
|
||||
assert.equal(
|
||||
residentModelIdMatches(
|
||||
"Qwen3-8B-Q4_K_M",
|
||||
"/srv/models/Llama-3-8B-Q4_K_M.gguf",
|
||||
null,
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
residentModelIdMatches(
|
||||
"unsloth/Qwen3-8B-GGUF",
|
||||
"/mnt/old-cache/models--unsloth--Llama-3-GGUF/snapshots/abc123",
|
||||
"unsloth/Llama-3-GGUF",
|
||||
),
|
||||
false,
|
||||
);
|
||||
assert.equal(residentModelIdMatches(null, "/srv/models/x.gguf"), false);
|
||||
assert.equal(residentModelIdMatches("Qwen3-8B-Q4_K_M"), false);
|
||||
});
|
||||
|
||||
test("a shared filename or folder name never marks a row resident", () => {
|
||||
// Two loose GGUFs with the same filename in different folders collapse onto one
|
||||
// public id, so a stem can only say "one of these", never which.
|
||||
const loaded = "/srv/models/alpha/model.gguf";
|
||||
const other = "/srv/models/beta/model.gguf";
|
||||
assert.equal(publicModelId(loaded), publicModelId(other));
|
||||
assert.equal(residentModelIdMatches(publicModelId(loaded), other, other), false);
|
||||
// The loadable identifier names exactly one of them.
|
||||
assert.equal(residentModelIdMatches(loaded, loaded, loaded), true);
|
||||
assert.equal(residentModelIdMatches(loaded, other, other), false);
|
||||
|
||||
// Same collapse one level up: two model directories sharing a basename.
|
||||
const loadedDir = "/srv/lmstudio/publisher-a/Llama-3-8B-GGUF";
|
||||
const otherDir = "/srv/models/publisher-b/Llama-3-8B-GGUF";
|
||||
assert.equal(publicModelId(loadedDir), publicModelId(otherDir));
|
||||
assert.equal(
|
||||
residentModelIdMatches(publicModelId(loadedDir), otherDir, otherDir),
|
||||
false,
|
||||
);
|
||||
|
||||
// A cache snapshot still collapses onto its repo id, which names one model.
|
||||
assert.equal(
|
||||
residentModelIdMatches(
|
||||
"unsloth/Qwen3-8B-GGUF",
|
||||
"/mnt/old-cache/models--unsloth--Qwen3-8B-GGUF/snapshots/abc123",
|
||||
null,
|
||||
),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test("Ollama link paths are recognised the way the resolver excludes them", () => {
|
||||
// core/inference/local_model_resolver.py refuses any path with these segments.
|
||||
assert.equal(
|
||||
isOllamaLinkPath("/home/u/.ollama/models/.studio_links/q/qwen3-Q4_K_M.gguf"),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isOllamaLinkPath("/home/u/.cache/unsloth/ollama_links/ab12/llama3.gguf"),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isOllamaLinkPath("C:\\Users\\u\\.ollama\\models\\.studio_links\\q\\a.gguf"),
|
||||
true,
|
||||
);
|
||||
// Only those exact segments, not a directory that merely contains the name.
|
||||
assert.equal(isOllamaLinkPath("/srv/studio_links_backup/a.gguf"), false);
|
||||
assert.equal(isOllamaLinkPath("/srv/models/Qwen3-8B-Q4_K_M.gguf"), false);
|
||||
assert.equal(isOllamaLinkPath("unsloth/Qwen3-8B-GGUF"), false);
|
||||
assert.equal(isOllamaLinkPath(null), false);
|
||||
});
|
||||
|
||||
test("a standalone gguf keeps one settings identity across surfaces", () => {
|
||||
const loose = {
|
||||
kind: "local",
|
||||
path: "/srv/models/Qwen3-8B-Q4_K_M.gguf",
|
||||
// What hub/services/models/common.py emits for a single scanned file.
|
||||
formatVariant: "Q4_K_M",
|
||||
} as LocalInventoryRow;
|
||||
// The Chat picker opens the same file with no variant, so the Hub row must not
|
||||
// adopt the filename-derived label or the two edit different configs.
|
||||
assert.equal(settingsGgufVariantForRow(loose), null);
|
||||
|
||||
// A GGUF directory still has a variant slot for the quant lookup to fill.
|
||||
const repoDir = {
|
||||
kind: "local",
|
||||
path: "/srv/models/Qwen3-8B-GGUF",
|
||||
formatVariant: null,
|
||||
} as LocalInventoryRow;
|
||||
assert.equal(settingsGgufVariantForRow(repoDir), null);
|
||||
const lmStudioDir = {
|
||||
kind: "local",
|
||||
path: "/srv/lmstudio/Qwen3-8B-GGUF",
|
||||
formatVariant: "Q8_0",
|
||||
} as LocalInventoryRow;
|
||||
assert.equal(settingsGgufVariantForRow(lmStudioDir), "Q8_0");
|
||||
|
||||
// Cached repo rows are unaffected (cache_inventory.py never sets one).
|
||||
const cached = { kind: "cache", formatVariant: null } as CachedInventoryRow;
|
||||
assert.equal(settingsGgufVariantForRow(cached), null);
|
||||
});
|
||||
|
||||
// The one-time backfill re-reads listPerModelConfigs() to pick up a save that
|
||||
// landed while the override fetch was in flight, and matches on the folded
|
||||
// identity. That is only unambiguous because storage holds one record per model,
|
||||
// so these pin that rule rather than the backfill.
|
||||
test("importing the legacy load settings never doubles up a model", () => {
|
||||
// The typed casing in unsloth_load_settings names the model the v2 record
|
||||
// already holds, so the import has to leave it alone rather than add a second
|
||||
// record the picker would prefer and the backfill would not.
|
||||
assert.deepEqual(listPerModelConfigs().length, 1);
|
||||
assert.deepEqual(storedKeys(), [REPO_KEY]);
|
||||
assert.equal(
|
||||
resolveInitialConfig("unsloth/repo-gguf", "q4_k_m").config
|
||||
.customContextLength,
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test("two spellings of one model id keep a single stored record", () => {
|
||||
store.clear();
|
||||
savePerModelConfig("Unsloth/Repo-GGUF", "Q4_K_M", config(4096));
|
||||
savePerModelConfig("unsloth/repo-gguf", "q4_k_m", config(32768, "q8_0"));
|
||||
|
||||
assert.deepEqual(storedKeys(), [REPO_KEY]);
|
||||
const listed = listPerModelConfigs();
|
||||
assert.equal(listed.length, 1);
|
||||
assert.equal(listed[0]?.config.maxSeqLength, 32768);
|
||||
// What the picker applies and the only thing the backfill can see agree.
|
||||
assert.equal(
|
||||
resolveInitialConfig("Unsloth/Repo-GGUF", "Q4_K_M").config.maxSeqLength,
|
||||
32768,
|
||||
);
|
||||
});
|
||||
|
||||
test("two spellings of one Windows path keep a single stored record", () => {
|
||||
store.clear();
|
||||
savePerModelConfig("C:\\Models\\Foo.gguf", null, config(4096));
|
||||
savePerModelConfig("c:/models/foo.gguf", null, config(32768, "q8_0"));
|
||||
|
||||
assert.deepEqual(storedKeys(), ['v2:["c:/models/foo.gguf",""]']);
|
||||
assert.equal(listPerModelConfigs().length, 1);
|
||||
});
|
||||
|
||||
test("a POSIX path is case sensitive, so its two spellings stay separate", () => {
|
||||
store.clear();
|
||||
savePerModelConfig("/models/Foo.gguf", null, config(4096));
|
||||
savePerModelConfig("/models/foo.gguf", null, config(32768, "q8_0"));
|
||||
|
||||
assert.equal(storedKeys().length, 2);
|
||||
assert.equal(
|
||||
resolveInitialConfig("/models/Foo.gguf", null).config.maxSeqLength,
|
||||
4096,
|
||||
);
|
||||
});
|
||||
|
||||
// Every answer below is the one split_quant_suffix in
|
||||
// studio/backend/utils/openai_auto_switch_settings.py gives for the same key. The
|
||||
// backfill folds a stored key with this before comparing it against the server's,
|
||||
// so a suffix this splits and the backend does not collapses two models onto one
|
||||
// key on the browser side only.
|
||||
const CASES: [string, [string, string] | null][] = [
|
||||
// A known quant label, with and without the optional bpw modifier.
|
||||
["org/Repo-GGUF:Q4_K_M", ["org/Repo-GGUF", "Q4_K_M"]],
|
||||
["org/Repo-GGUF:IQ4_XS-3.53bpw", ["org/Repo-GGUF", "IQ4_XS-3.53bpw"]],
|
||||
["org/Repo-GGUF:UD-Q4_K_XL", ["org/Repo-GGUF", "UD-Q4_K_XL"]],
|
||||
// A .gguf with no quant token in its name is labelled by its stem, and storage
|
||||
// lowercases the label while the scanner keeps the filename's casing.
|
||||
["/models/CustomModel.gguf:custommodel", ["/models/CustomModel.gguf", "custommodel"]],
|
||||
["/models/CustomModel.gguf:CustomModel", ["/models/CustomModel.gguf", "CustomModel"]],
|
||||
["C:\\models\\CustomModel.gguf:custommodel", ["C:\\models\\CustomModel.gguf", "custommodel"]],
|
||||
// A shard suffix is not part of the label.
|
||||
[
|
||||
"/models/Custom-00001-of-00003.gguf:custom",
|
||||
["/models/Custom-00001-of-00003.gguf", "custom"],
|
||||
],
|
||||
["/models/Custom-00001-of-00003.gguf:custom-00001-of-00003", null],
|
||||
// An extensionless .gguf still has a label.
|
||||
["/models/.gguf:gguf", ["/models/.gguf", "gguf"]],
|
||||
// A quant token inside the filename wins over the stem.
|
||||
["/models/tinyllama-Q4_K_M.gguf:q4_k_m", ["/models/tinyllama-Q4_K_M.gguf", "q4_k_m"]],
|
||||
["/models/tinyllama-Q4_K_M.gguf:tinyllama-q4_k_m", null],
|
||||
// Only the basename is labelled, never the directories above it.
|
||||
[
|
||||
"/models/dir/CustomModel.gguf:custommodel",
|
||||
["/models/dir/CustomModel.gguf", "custommodel"],
|
||||
],
|
||||
["/models/dir/CustomModel.gguf:dir/custommodel", null],
|
||||
// A colon is legal in a POSIX filename. Neither of these is a variant, and
|
||||
// reading them as one folds two real files onto a single key.
|
||||
["/models/foo:Bar.gguf", null],
|
||||
["/models/foo:bar.gguf", null],
|
||||
["/models/llama.gguf:Bar.gguf", null],
|
||||
["/models/llama.gguf:bar.gguf", null],
|
||||
["/models/CustomModel.gguf:othermodel", null],
|
||||
["/models/model.gguf:notalabel", null],
|
||||
["/models/plain.gguf:plain:extra", null],
|
||||
// A Windows drive letter is not a separator either.
|
||||
["C:\\models\\foo.gguf", null],
|
||||
["C:/models/foo.gguf", null],
|
||||
// Nothing to split.
|
||||
["org/Repo-GGUF", null],
|
||||
["/models/foo.gguf", null],
|
||||
["org/Repo:", null],
|
||||
[":Q4_K_M", null],
|
||||
];
|
||||
|
||||
test("splitQuantSuffix answers exactly as the backend's split_quant_suffix", () => {
|
||||
for (const [value, expected] of CASES) {
|
||||
assert.deepEqual(splitQuantSuffix(value), expected, value);
|
||||
}
|
||||
});
|
||||
|
||||
test("a .gguf filename carrying a colon is not folded into a variant", () => {
|
||||
// Two real, distinct files: POSIX allows a colon in a name and is case
|
||||
// sensitive, so the one-time backfill has to keep their settings apart. The
|
||||
// variant half of an override key is stored lowercased, so folding these makes
|
||||
// one key and strands whichever file the backfill reaches second.
|
||||
const upper = "/models/llama.gguf:Bar.gguf";
|
||||
const lower = "/models/llama.gguf:bar.gguf";
|
||||
assert.equal(splitQuantSuffix(upper), null);
|
||||
assert.equal(splitQuantSuffix(lower), null);
|
||||
assert.notEqual(modelStorageKey(upper, null), modelStorageKey(lower, null));
|
||||
});
|
||||
|
|
@ -3,7 +3,9 @@
|
|||
"target": "ES2022",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"module": "ESNext",
|
||||
"types": ["node"],
|
||||
// vite/client so a test may import a src module that (transitively) reads
|
||||
// import.meta.env; without it those reads fail to typecheck here only.
|
||||
"types": ["node", "vite/client"],
|
||||
"skipLibCheck": true,
|
||||
|
||||
"moduleResolution": "bundler",
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -5,7 +5,10 @@ from pathlib import Path
|
|||
|
||||
REPO = Path(__file__).resolve().parents[2]
|
||||
SETTINGS_DIALOG = REPO / "studio/frontend/src/features/settings/settings-dialog.tsx"
|
||||
API_MONITOR = REPO / "studio/frontend/src/features/settings/components/api-monitor-console.tsx"
|
||||
# The monitor moved onto its own page and Settings links to it; the shrink
|
||||
# contract still applies to both surfaces.
|
||||
API_MONITOR_PAGE = REPO / "studio/frontend/src/features/api-monitor/api-monitor-page.tsx"
|
||||
MONITOR_LINK = REPO / "studio/frontend/src/features/settings/components/monitor-link.tsx"
|
||||
GENERAL_TAB = REPO / "studio/frontend/src/features/settings/tabs/general-tab.tsx"
|
||||
|
||||
|
||||
|
|
@ -16,15 +19,22 @@ def test_dialog_content_can_shrink_inside_the_dialog_grid():
|
|||
|
||||
|
||||
def test_api_monitor_entries_and_expanded_text_can_shrink():
|
||||
source = API_MONITOR.read_text(encoding = "utf-8")
|
||||
assert (
|
||||
'<article className="min-w-0 rounded-lg border border-border/70 bg-background">' in source
|
||||
)
|
||||
assert (
|
||||
'<section className="flex min-w-0 flex-col rounded-lg border border-border/70 bg-background">'
|
||||
in source
|
||||
)
|
||||
assert source.count('className="max-h-44 overflow-auto whitespace-pre-wrap break-words') == 2
|
||||
source = API_MONITOR_PAGE.read_text(encoding = "utf-8")
|
||||
# Rows and the detail pane sit in flex parents, so they need min-w-0 or a long
|
||||
# model id pushes the layout wider than the viewport.
|
||||
assert '"flex w-full min-w-0 flex-col gap-1 border-b border-border/50' in source
|
||||
assert '<section className="flex min-w-0 flex-col gap-1.5">' in source
|
||||
# Prompt and reply are unbounded user text: height-capped, scrollable, wrapped.
|
||||
assert "max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-lg bg-muted/50" in source
|
||||
# A model id or path has no spaces to wrap on, so it needs break-all.
|
||||
assert 'className="min-w-0 break-all font-mono' in source
|
||||
|
||||
|
||||
def test_settings_monitor_link_can_shrink():
|
||||
source = MONITOR_LINK.read_text(encoding = "utf-8")
|
||||
assert "flex w-full min-w-0 items-center gap-3" in source
|
||||
# The summary line carries a model id, so it truncates instead of widening.
|
||||
assert '<span className="truncate text-xs text-muted-foreground">' in source
|
||||
|
||||
|
||||
def test_embedding_model_controls_stack_on_the_narrowest_viewports():
|
||||
|
|
|
|||
|
|
@ -132,33 +132,45 @@ def test_usage_examples_has_no_duplicate_auto_switch_control():
|
|||
assert "<ModelAutoSwitchSection />" in tab
|
||||
|
||||
|
||||
API_MONITOR_TSX = SETTINGS / "components/api-monitor-console.tsx"
|
||||
# The monitor moved onto its own page; Settings keeps configuration and links
|
||||
# across. These contracts follow the behaviour, not the old file.
|
||||
API_MONITOR_TSX = REPO / "studio/frontend/src/features/api-monitor/api-monitor-page.tsx"
|
||||
# The lifecycle labels live in their own module: the overlay is mounted from
|
||||
# __root.tsx, so importing them from the page pulled it into the eager bundle.
|
||||
API_MONITOR_LIFECYCLE_TS = REPO / "studio/frontend/src/features/api-monitor/lifecycle.ts"
|
||||
MONITOR_LINK_TSX = SETTINGS / "components/monitor-link.tsx"
|
||||
|
||||
|
||||
def test_api_monitor_pages_five_at_a_time():
|
||||
# The backend retains 50 terminal entries; the console used to dump them all at once.
|
||||
def test_api_monitor_history_does_not_reorder_under_the_reader():
|
||||
# The backend keeps 50 terminal entries and moves one to the front as it
|
||||
# finishes. The console froze ids while paging; the page pauses the poll instead,
|
||||
# holding the whole list still while a payload is read.
|
||||
src = API_MONITOR_TSX.read_text(encoding = "utf-8")
|
||||
assert "const PAGE_SIZE = 5;" in src
|
||||
assert "ordered.slice(" in src
|
||||
# Paging back must freeze the id order, or live traffic reorders history under it.
|
||||
assert "frozenIds" in src
|
||||
assert "setFrozenIds((prev) => prev ?? entries.map((entry) => entry.id))" in src
|
||||
assert "paused" in src
|
||||
assert "setPaused" in src
|
||||
# Filters and search are what keep 50 rows usable without paging.
|
||||
assert "filterEntries(" in src
|
||||
assert "STATUS_FILTERS" in src
|
||||
|
||||
|
||||
def test_api_monitor_renders_lifecycle_rows():
|
||||
src = API_MONITOR_TSX.read_text(encoding = "utf-8")
|
||||
assert "function LifecycleEntry(" in src
|
||||
assert 'entry.kind === "lifecycle"' in src
|
||||
labels = API_MONITOR_LIFECYCLE_TS.read_text(encoding = "utf-8")
|
||||
assert "export function isLifecycleEntry(" in labels
|
||||
assert 'entry.kind === "lifecycle"' in labels
|
||||
for label in ("Loading model", "Model loaded", "Model unloaded"):
|
||||
assert label in src
|
||||
# Lifecycle rows have no prompt/reply to fetch.
|
||||
assert "isLifecycle(entry) || !expandedIds.has(entry.id)" in src
|
||||
assert label in labels
|
||||
# A lifecycle row has no prompt or reply, so it is not selectable for detail.
|
||||
assert "if (isLifecycleEntry(entry)) {" in src
|
||||
assert 'from "./lifecycle"' in src
|
||||
|
||||
|
||||
def test_auto_switch_section_sits_above_the_monitor():
|
||||
def test_auto_switch_section_sits_above_the_usage_examples():
|
||||
tab = API_KEYS_TAB_TSX.read_text(encoding = "utf-8")
|
||||
assert tab.index("<ModelAutoSwitchSection />") < tab.index("<ApiMonitorConsole />")
|
||||
assert tab.index("<ApiMonitorConsole />") < tab.index("<UsageExamples")
|
||||
# The console became a link out; ordering still puts configuration ahead of the
|
||||
# examples that depend on it.
|
||||
assert tab.index("<MonitorLink />") < tab.index("<ModelAutoSwitchSection />")
|
||||
assert tab.index("<ModelAutoSwitchSection />") < tab.index("<UsageExamples")
|
||||
|
||||
|
||||
AUTO_SWITCH_TSX = SETTINGS / "components/model-auto-switch-section.tsx"
|
||||
|
|
@ -166,7 +178,7 @@ EN_TS = REPO / "studio/frontend/src/i18n/locales/en.ts"
|
|||
|
||||
|
||||
def test_api_monitor_renders_download_rows():
|
||||
src = API_MONITOR_TSX.read_text(encoding = "utf-8")
|
||||
src = API_MONITOR_LIFECYCLE_TS.read_text(encoding = "utf-8")
|
||||
assert 'entry.event === "download"' in src
|
||||
for label in ("Downloading model", "Model downloaded", "Model download failed"):
|
||||
assert label in src
|
||||
|
|
@ -183,6 +195,12 @@ def test_monitor_can_unload_the_loaded_model():
|
|||
assert "unloadModel({ model_path: checkpoint })" in src
|
||||
|
||||
|
||||
def test_settings_still_reaches_the_monitor():
|
||||
# The console is gone, so Settings must still have a way through to it.
|
||||
link = MONITOR_LINK_TSX.read_text(encoding = "utf-8")
|
||||
assert 'to: "/api-monitor"' in link
|
||||
|
||||
|
||||
def test_auto_download_toggle_is_gated_on_auto_switch():
|
||||
# Downloading what auto-switch cannot load fetches gigabytes nothing can serve.
|
||||
src = AUTO_SWITCH_TSX.read_text(encoding = "utf-8")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue