studio: apply saved per-model settings on API loads, add API monitor
Per-model settings were mirrored to the server as two fields only, llama_extra_args and max_seq_length. Everything else lived in browser localStorage, so a model loaded by an API request came up with app defaults for context length, KV cache dtype, speculative decoding, tensor parallel and GPU placement. Store the full config server side and map it onto the same LoadRequest the picker builds, so a remote load and a picker load of the same model produce the same command line. Entries are keyed per quant, falling back to the bare repo id so existing entries keep resolving. Also moves the API monitor out of the settings tab: a full page at /api-monitor, plus a floating panel that opens itself when traffic arrives, and a settings page per model reachable from the Hub.
This commit is contained in:
Unsloth
parent
e7d047a4ee
commit
9ec6c8bb5a
31 changed files with 2988 additions and 444 deletions
|
|
@ -269,9 +269,20 @@ class ApiMonitor:
|
|||
if entry.status == "running" and (subject is None or entry.subject == subject)
|
||||
)
|
||||
|
||||
def clear(self) -> None:
|
||||
def clear(self, *, subject: Optional[str] = None) -> None:
|
||||
"""Drop recorded entries. ``subject`` limits the wipe to one caller's.
|
||||
|
||||
Every other read on this class is subject-scoped, so an unscoped clear
|
||||
would let one user erase another's history (and zero their active count
|
||||
mid-generation). Callers that genuinely mean "everything" pass None.
|
||||
"""
|
||||
with self._lock:
|
||||
self._entries.clear()
|
||||
if subject is None:
|
||||
self._entries.clear()
|
||||
return
|
||||
self._entries = deque(
|
||||
entry for entry in self._entries if entry.subject != subject
|
||||
)
|
||||
|
||||
def _find_locked(self, entry_id: str) -> Optional[ApiMonitorEntry]:
|
||||
for entry in self._entries:
|
||||
|
|
|
|||
|
|
@ -3581,6 +3581,7 @@ async def _maybe_auto_switch_model(
|
|||
get_openai_auto_switch_enabled,
|
||||
get_auto_unload_idle_seconds,
|
||||
get_model_override,
|
||||
model_override_load_kwargs,
|
||||
)
|
||||
from core.inference.local_model_resolver import resolve_local_gguf
|
||||
from core.inference.llama_keepwarm import (
|
||||
|
|
@ -3716,13 +3717,35 @@ async def _maybe_auto_switch_model(
|
|||
if _already_serving():
|
||||
_record_serving_alias()
|
||||
return
|
||||
# Apply this model's saved launch flags so the swap honors the config.
|
||||
override = get_model_override(override_id)
|
||||
# Apply this model's saved launch config so an API-driven swap
|
||||
# loads it exactly as the picker would: context, KV dtype,
|
||||
# speculative decoding, chat template and GPU placement, not
|
||||
# just the two legacy flags. Look the config up under the
|
||||
# variant-qualified id first (two quants of one repo can carry
|
||||
# different configs), then the bare advertised id, then the
|
||||
# concrete load path, so a config saved under any of the names
|
||||
# this model is known by is found.
|
||||
override = {}
|
||||
for override_key in (
|
||||
f"{override_id}:{variant}" if variant else None,
|
||||
override_id,
|
||||
target_id,
|
||||
):
|
||||
if not override_key:
|
||||
continue
|
||||
override = get_model_override(override_key)
|
||||
if override:
|
||||
break
|
||||
load_kwargs = {"model_path": target_id, "gguf_variant": variant}
|
||||
if override.get("llama_extra_args") is not None:
|
||||
load_kwargs["llama_extra_args"] = override["llama_extra_args"]
|
||||
if override.get("max_seq_length") is not None:
|
||||
load_kwargs["max_seq_length"] = override["max_seq_length"]
|
||||
load_kwargs.update(
|
||||
model_override_load_kwargs(
|
||||
override,
|
||||
# variant is set for every GGUF the resolver returns; the
|
||||
# reload-stash path carries the quant it froze.
|
||||
is_gguf = bool(variant)
|
||||
or target_id.lower().endswith(".gguf"),
|
||||
)
|
||||
)
|
||||
# Reuse the load impl so its dedup, tensor fallback, and threading
|
||||
# apply. Call the impl directly: we already hold the lifecycle gate
|
||||
# the /load route would otherwise take, so the route would deadlock.
|
||||
|
|
@ -5703,6 +5726,22 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)):
|
|||
}
|
||||
|
||||
|
||||
@studio_router.delete("/monitor")
|
||||
async def clear_api_monitor(current_subject: str = Depends(get_current_subject)):
|
||||
"""Drop this caller's recorded API history so a debugging session starts clean.
|
||||
|
||||
Scoped to the current subject, like every read on the monitor: an unscoped
|
||||
wipe would erase another user's history and zero their active-request count
|
||||
while their generation is still streaming.
|
||||
|
||||
The caller's own in-flight requests are dropped from the log too; they keep
|
||||
streaming to their client, they just stop being reported here (a later append
|
||||
re-adds nothing, since the entry id no longer resolves).
|
||||
"""
|
||||
api_monitor.clear(subject = current_subject)
|
||||
return {"cleared": True}
|
||||
|
||||
|
||||
@studio_router.get("/monitor/{entry_id}")
|
||||
async def get_api_monitor_entry(entry_id: str, current_subject: str = Depends(get_current_subject)):
|
||||
"""Return full prompt/reply details for one OpenAI-compatible API request."""
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ from utils.helper_precache_settings import (
|
|||
helper_model_disabled_by_env,
|
||||
set_helper_precache_enabled,
|
||||
)
|
||||
from picker.schemas import MAX_CHAT_TEMPLATE_BYTES
|
||||
from utils.coding_agents import CODING_AGENTS, detect_installed_coding_agents
|
||||
from utils.openai_auto_switch_settings import (
|
||||
DEFAULT_AUTO_UNLOAD_KEEP_KV,
|
||||
|
|
@ -126,11 +127,51 @@ class OpenAIAutoSwitchResponse(BaseModel):
|
|||
|
||||
|
||||
class ModelOverridePayload(BaseModel):
|
||||
model_id: str = Field(..., min_length = 1)
|
||||
llama_extra_args: list[str] = Field(default_factory = list)
|
||||
"""One model's saved launch config, applied when the API loads that model.
|
||||
|
||||
Everything past ``model_id`` is optional and omitted means "app default", so a
|
||||
payload carrying only ``model_id`` clears the entry. The bounds here mirror
|
||||
``LoadRequest`` so a bad value is rejected at the boundary instead of being
|
||||
silently dropped by the normalizer; the enum-ish fields (KV dtype, speculative
|
||||
mode) are left to it, since their valid sets follow the llama.cpp build.
|
||||
"""
|
||||
|
||||
model_id: str = Field(..., min_length = 1, max_length = 512)
|
||||
# None means "leave the stored value alone": the settings UI has no control
|
||||
# for launch flags, so a save from it must not wipe flags set through this
|
||||
# API. An explicit [] clears them (that is how "forget this model" arrives).
|
||||
llama_extra_args: Optional[list[str]] = None
|
||||
# ge=1: 0 is not a valid sequence length, and the setter drops a falsy value,
|
||||
# so reject it at the boundary instead of accepting then silently discarding it.
|
||||
max_seq_length: Optional[int] = Field(default = None, ge = 1, le = 1048576)
|
||||
custom_context_length: Optional[int] = Field(default = None, ge = 1, le = 1048576)
|
||||
kv_cache_dtype: Optional[str] = Field(default = None, max_length = 32)
|
||||
speculative_type: Optional[str] = Field(default = None, max_length = 32)
|
||||
spec_draft_n_max: Optional[int] = Field(default = None, ge = 1, le = 16)
|
||||
tensor_parallel: bool = False
|
||||
# Validated in bytes below, not by max_length: pydantic counts characters,
|
||||
# so a multi-byte template could pass here and then be silently dropped by
|
||||
# the normalizer (which measures UTF-8) while the request still returned 200.
|
||||
chat_template_override: Optional[str] = None
|
||||
gpu_memory_mode: Optional[Literal["auto", "manual"]] = None
|
||||
# -1 is Auto (llama.cpp --fit sizes the offload); the normalizer treats it as unset.
|
||||
gpu_layers: Optional[int] = Field(default = None, ge = -1, le = 1024)
|
||||
n_cpu_moe: Optional[int] = Field(default = None, ge = 0, le = 1024)
|
||||
gpu_ids: Optional[list[int]] = None
|
||||
|
||||
|
||||
@field_validator("chat_template_override")
|
||||
@classmethod
|
||||
def _limit_chat_template_bytes(cls, value: Optional[str]) -> Optional[str]:
|
||||
# Mirrors LoadRequest.normalize_blank_chat_template_override so the same
|
||||
# template is accepted or rejected identically on both paths.
|
||||
if value is None:
|
||||
return None
|
||||
if len(value.encode("utf-8")) > MAX_CHAT_TEMPLATE_BYTES:
|
||||
raise ValueError(
|
||||
f"Chat template exceeds the {MAX_CHAT_TEMPLATE_BYTES}-byte limit."
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
class ModelOverridesResponse(BaseModel):
|
||||
|
|
@ -289,12 +330,40 @@ def update_openai_auto_switch_override(
|
|||
payload: ModelOverridePayload, current_subject: str = Depends(get_current_subject)
|
||||
) -> ModelOverridesResponse:
|
||||
from core.inference.llama_server_args import validate_extra_args
|
||||
from utils.openai_auto_switch_settings import get_model_override
|
||||
try:
|
||||
extra_args = validate_extra_args(payload.llama_extra_args)
|
||||
# A payload carrying only model_id is the documented "remove", so it
|
||||
# wipes everything. Otherwise it is a real save, and omitted launch flags
|
||||
# are carried over from the stored entry (the settings UI cannot express
|
||||
# them and must not delete them).
|
||||
requested_extra_args = payload.llama_extra_args
|
||||
saved_fields = payload.model_dump(
|
||||
exclude = {"model_id", "llama_extra_args"}, exclude_none = True
|
||||
)
|
||||
is_removal = not payload.tensor_parallel and not {
|
||||
key: value
|
||||
for key, value in saved_fields.items()
|
||||
if key != "tensor_parallel"
|
||||
}
|
||||
if requested_extra_args is None and not is_removal:
|
||||
requested_extra_args = get_model_override(payload.model_id).get(
|
||||
"llama_extra_args"
|
||||
)
|
||||
extra_args = validate_extra_args(requested_extra_args)
|
||||
set_model_override(
|
||||
payload.model_id,
|
||||
llama_extra_args = extra_args,
|
||||
max_seq_length = payload.max_seq_length,
|
||||
custom_context_length = payload.custom_context_length,
|
||||
kv_cache_dtype = payload.kv_cache_dtype,
|
||||
speculative_type = payload.speculative_type,
|
||||
spec_draft_n_max = payload.spec_draft_n_max,
|
||||
tensor_parallel = payload.tensor_parallel,
|
||||
chat_template_override = payload.chat_template_override,
|
||||
gpu_memory_mode = payload.gpu_memory_mode,
|
||||
gpu_layers = payload.gpu_layers,
|
||||
n_cpu_moe = payload.n_cpu_moe,
|
||||
gpu_ids = payload.gpu_ids,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise log_and_http_error(
|
||||
|
|
|
|||
|
|
@ -258,3 +258,34 @@ def test_api_monitor_append_reply_exact_cap_then_more_marks_truncated():
|
|||
monitor.append_reply(entry_id, "y")
|
||||
reply = monitor.snapshot()[0]["reply"]
|
||||
assert len(reply) == m._MAX_REPLY_CHARS and reply.endswith("...")
|
||||
|
||||
|
||||
def test_api_monitor_clear_is_scoped_to_one_subject():
|
||||
# Every other read on the monitor is subject-scoped. An unscoped clear from
|
||||
# the route would let one caller erase another's history and zero their
|
||||
# active count in the middle of a generation.
|
||||
monitor = ApiMonitor(max_entries = 4)
|
||||
alice = monitor.start(
|
||||
endpoint = "/v1/chat/completions",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "alice prompt",
|
||||
subject = "alice",
|
||||
)
|
||||
bob = monitor.start(
|
||||
endpoint = "/v1/chat/completions",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "bob prompt",
|
||||
subject = "bob",
|
||||
)
|
||||
|
||||
monitor.clear(subject = "alice")
|
||||
assert monitor.snapshot(subject = "alice") == []
|
||||
assert [entry["id"] for entry in monitor.snapshot(subject = "bob")] == [bob]
|
||||
assert monitor.active_count(subject = "bob") == 1
|
||||
assert monitor.get(alice, subject = "alice") is None
|
||||
|
||||
# Passing no subject is the explicit "everything" path.
|
||||
monitor.clear()
|
||||
assert monitor.snapshot(subject = "bob") == []
|
||||
|
|
|
|||
|
|
@ -3819,3 +3819,177 @@ def test_env_idle_below_floor_is_clamped(monkeypatch):
|
|||
assert settings.get_auto_unload_idle_seconds() == 600
|
||||
monkeypatch.delenv(settings.MODEL_IDLE_TTL_ENV_VAR)
|
||||
assert settings.get_auto_unload_idle_seconds() == 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-model launch config: normalization, LoadRequest mapping, key resolution.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_normalize_model_override_drops_unusable_fields_and_keeps_the_rest():
|
||||
# A stale field must not cost the user the whole config, so bad values are
|
||||
# dropped one by one rather than rejecting the payload.
|
||||
entry = settings.normalize_model_override(
|
||||
{
|
||||
"max_seq_length": 8192,
|
||||
"kv_cache_dtype": "not_a_dtype",
|
||||
"speculative_type": "mtp",
|
||||
"spec_draft_n_max": 999, # out of range for the MTP draft count
|
||||
"gpu_memory_mode": "auto", # only "manual" is a real override
|
||||
"gpu_layers": -1, # -1 is Auto, which is already the default
|
||||
"n_cpu_moe": 0,
|
||||
"gpu_ids": [1, 1, 0, "2", -5],
|
||||
"tensor_parallel": False,
|
||||
"llama_extra_args": [],
|
||||
}
|
||||
)
|
||||
assert entry == {
|
||||
"max_seq_length": 8192,
|
||||
"speculative_type": "mtp",
|
||||
"gpu_ids": [1, 0, 2],
|
||||
}
|
||||
|
||||
|
||||
def test_normalize_model_override_rejects_oversized_chat_template():
|
||||
small = settings.normalize_model_override({"chat_template_override": "{{ bos }}"})
|
||||
assert small["chat_template_override"] == "{{ bos }}"
|
||||
# The limit is bytes, not characters: a multi-byte template just under the
|
||||
# character limit can still be over the byte limit.
|
||||
huge = "é" * settings.MAX_CHAT_TEMPLATE_OVERRIDE_BYTES
|
||||
assert "chat_template_override" not in settings.normalize_model_override(
|
||||
{"chat_template_override": huge}
|
||||
)
|
||||
|
||||
|
||||
def test_spec_draft_n_max_only_stored_for_mtp_modes():
|
||||
mtp = settings.normalize_model_override(
|
||||
{"speculative_type": "mtp", "spec_draft_n_max": 4}
|
||||
)
|
||||
assert mtp["spec_draft_n_max"] == 4
|
||||
# A non-MTP mode ignores the draft count at load time, so storing it would
|
||||
# show the user an edit that never takes effect.
|
||||
ngram = settings.normalize_model_override(
|
||||
{"speculative_type": "ngram", "spec_draft_n_max": 4}
|
||||
)
|
||||
assert "spec_draft_n_max" not in ngram
|
||||
|
||||
|
||||
def test_resolve_fit_max_seq_length_hands_sizing_to_fit_under_manual_auto_layers():
|
||||
# Manual GPU memory with Auto layers means llama.cpp --fit owns the context,
|
||||
# so the load sends the context pin (or 0), not the stored max seq length.
|
||||
override = {"gpu_memory_mode": "manual", "max_seq_length": 8192}
|
||||
assert settings.resolve_fit_max_seq_length(override, is_gguf = True) == 0
|
||||
assert (
|
||||
settings.resolve_fit_max_seq_length(
|
||||
{**override, "custom_context_length": 4096}, is_gguf = True
|
||||
)
|
||||
== 4096
|
||||
)
|
||||
# Pinning the layer count takes --fit back out of the picture.
|
||||
assert (
|
||||
settings.resolve_fit_max_seq_length({**override, "gpu_layers": 20}, is_gguf = True)
|
||||
== 8192
|
||||
)
|
||||
# Not a GGUF, so none of this applies.
|
||||
assert settings.resolve_fit_max_seq_length(override, is_gguf = False) == 8192
|
||||
|
||||
|
||||
def test_model_override_load_kwargs_gates_gpu_placement_on_gguf():
|
||||
override = {
|
||||
"max_seq_length": 4096,
|
||||
"kv_cache_dtype": "q8_0",
|
||||
"tensor_parallel": True,
|
||||
"gpu_memory_mode": "manual",
|
||||
"gpu_layers": 20,
|
||||
"n_cpu_moe": 3,
|
||||
"gpu_ids": [0, 1],
|
||||
}
|
||||
gguf = settings.model_override_load_kwargs(override, is_gguf = True)
|
||||
assert gguf["cache_type_kv"] == "q8_0"
|
||||
assert gguf["tensor_parallel"] is True
|
||||
assert gguf["gpu_layers"] == 20
|
||||
assert gguf["gpu_ids"] == [0, 1]
|
||||
|
||||
# A safetensors model loads through HF auto-placement; inheriting a GGUF GPU
|
||||
# pin here would silently change where the weights land.
|
||||
safetensors = settings.model_override_load_kwargs(override, is_gguf = False)
|
||||
assert safetensors["max_seq_length"] == 4096
|
||||
assert "gpu_layers" not in safetensors
|
||||
assert "gpu_ids" not in safetensors
|
||||
assert "n_cpu_moe" not in safetensors
|
||||
assert "gpu_memory_mode" not in safetensors
|
||||
|
||||
# Every key it produces has to be a real LoadRequest field, or the load call
|
||||
# raises TypeError at the moment the user's request arrives.
|
||||
LoadRequest(model_path = "unsloth/B-GGUF", **gguf)
|
||||
|
||||
|
||||
def test_auto_switch_prefers_variant_qualified_override(monkeypatch):
|
||||
# Settings are saved per quant, so Q4_K_M and Q8_0 of the same repo are
|
||||
# different entries; the bare repo id is only the fallback.
|
||||
backend = _FakeBackend(None)
|
||||
rec = _LoadRecorder(backend)
|
||||
_wire(
|
||||
monkeypatch,
|
||||
enabled = True,
|
||||
resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"),
|
||||
backend = backend,
|
||||
recorder = rec,
|
||||
)
|
||||
stored = {
|
||||
"unsloth/B-GGUF": {"max_seq_length": 1024},
|
||||
"unsloth/B-GGUF:Q4_K_M": {"max_seq_length": 8192, "gpu_layers": 20},
|
||||
}
|
||||
monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {}))
|
||||
|
||||
_run_hook("unsloth/B-GGUF")
|
||||
req = rec.calls[0]
|
||||
assert req.max_seq_length == 8192
|
||||
assert req.gpu_layers == 20
|
||||
|
||||
|
||||
def test_auto_switch_falls_back_to_bare_repo_override(monkeypatch):
|
||||
backend = _FakeBackend(None)
|
||||
rec = _LoadRecorder(backend)
|
||||
_wire(
|
||||
monkeypatch,
|
||||
enabled = True,
|
||||
resolves_to = ("unsloth/B-GGUF", "Q4_K_M", "unsloth/B-GGUF"),
|
||||
backend = backend,
|
||||
recorder = rec,
|
||||
)
|
||||
stored = {"unsloth/B-GGUF": {"max_seq_length": 1024}}
|
||||
monkeypatch.setattr(settings, "get_model_override", lambda mid: stored.get(mid, {}))
|
||||
|
||||
_run_hook("unsloth/B-GGUF")
|
||||
assert rec.calls[0].max_seq_length == 1024
|
||||
|
||||
|
||||
def test_override_route_preserves_launch_flags_across_a_settings_only_update(monkeypatch):
|
||||
# The settings page has no control for llama_extra_args, so saving from it
|
||||
# omits the field. Omitted must mean "leave it alone", or every save from the
|
||||
# UI would quietly wipe flags set elsewhere.
|
||||
import routes.settings as settings_route
|
||||
|
||||
_mock_override_store(monkeypatch)
|
||||
settings_route.update_openai_auto_switch_override(
|
||||
settings_route.ModelOverridePayload(
|
||||
model_id = "unsloth/B-GGUF", llama_extra_args = ["--flash-attn"]
|
||||
),
|
||||
"tester",
|
||||
)
|
||||
resp = settings_route.update_openai_auto_switch_override(
|
||||
settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", max_seq_length = 4096),
|
||||
"tester",
|
||||
)
|
||||
entry = resp.overrides["unsloth/B-GGUF"]
|
||||
assert entry["llama_extra_args"] == ["--flash-attn"]
|
||||
assert entry["max_seq_length"] == 4096
|
||||
|
||||
# An explicit empty list is how the UI says "forget this model", and with no
|
||||
# other fields left that removes the entry outright.
|
||||
gone = settings_route.update_openai_auto_switch_override(
|
||||
settings_route.ModelOverridePayload(model_id = "unsloth/B-GGUF", llama_extra_args = []),
|
||||
"tester",
|
||||
)
|
||||
assert "unsloth/B-GGUF" not in gone.overrides
|
||||
|
|
|
|||
|
|
@ -210,8 +210,211 @@ def set_openai_auto_switch(
|
|||
)
|
||||
|
||||
|
||||
# --- Per-model launch config -------------------------------------------------
|
||||
#
|
||||
# An override is the server-side twin of the UI's per-model config (the browser
|
||||
# localStorage map behind features/model-picker/model-config). The UI mirrors
|
||||
# every save here so a model loaded by an OpenAI-compatible API request gets the
|
||||
# same launch settings a user would get loading it from the picker; without this
|
||||
# the API path could only ever apply the two legacy fields below.
|
||||
#
|
||||
# Legacy entries hold just {llama_extra_args, max_seq_length}; every field is
|
||||
# optional and absent means "fall back to the app default", so old entries keep
|
||||
# loading correctly. A write is a full replace of the fields it expresses, so the
|
||||
# route carries `llama_extra_args` over when the payload omits it (the settings
|
||||
# UI has no control for launch flags and must not wipe them).
|
||||
#
|
||||
# Known gap: the picker resolves a couple of knobs as "per-model value, else the
|
||||
# user's global preference" -- GPU memory mode and speculative decoding, whose
|
||||
# globals live in browser localStorage. An override deliberately stores only an
|
||||
# explicit per-model choice (so the model keeps following later global changes),
|
||||
# and the server cannot see the globals at all. So for a model that follows the
|
||||
# global on one of those two, an API load falls back to the app default rather
|
||||
# than the user's global. Every other field matches the picker exactly.
|
||||
|
||||
# Mirrors _valid_cache_types in core/inference/llama_cpp.py.
|
||||
VALID_KV_CACHE_DTYPES = frozenset(
|
||||
{"f16", "bf16", "q8_0", "q4_0", "q4_1", "q5_0", "q5_1", "iq4_nl", "f32"}
|
||||
)
|
||||
# Canonical values plus the legacy spellings LoadRequest still accepts.
|
||||
VALID_SPECULATIVE_TYPES = frozenset(
|
||||
{
|
||||
"auto",
|
||||
"mtp",
|
||||
"ngram",
|
||||
"mtp+ngram",
|
||||
"off",
|
||||
"default",
|
||||
"draft-mtp",
|
||||
"ngram-mod",
|
||||
"ngram-simple",
|
||||
}
|
||||
)
|
||||
# Only these two consume spec_draft_n_max (mirrors MTP_SPECULATIVE_TYPES in the UI).
|
||||
MTP_SPECULATIVE_TYPES = frozenset({"mtp", "mtp+ngram", "draft-mtp"})
|
||||
VALID_GPU_MEMORY_MODES = frozenset({"auto", "manual"})
|
||||
|
||||
MAX_SEQ_LENGTH_CEILING = 1048576
|
||||
MAX_CHAT_TEMPLATE_OVERRIDE_BYTES = 65_536
|
||||
|
||||
|
||||
def _clean_str(value: Any, allowed: frozenset[str]) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
normalized = value.strip().lower()
|
||||
return normalized if normalized in allowed else None
|
||||
|
||||
|
||||
def _bounded_int(value: Any, *, minimum: int, maximum: int) -> Optional[int]:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if parsed < minimum or parsed > maximum:
|
||||
return None
|
||||
return parsed
|
||||
|
||||
|
||||
def normalize_model_override(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Validate one per-model launch config, dropping anything unusable.
|
||||
|
||||
Silently drops rather than raising: an override is a convenience mirror of the
|
||||
UI's config, so one stale field (a KV dtype this llama.cpp build lost, a GPU id
|
||||
from another host) must not block persisting the rest or fail the API load that
|
||||
reads it. ``validate_extra_args`` is the caller's job -- it lives in the
|
||||
llama_server_args allow-list module, which this one must not import.
|
||||
"""
|
||||
entry: dict[str, Any] = {}
|
||||
|
||||
extra_args = payload.get("llama_extra_args")
|
||||
if isinstance(extra_args, (list, tuple)) and extra_args:
|
||||
entry["llama_extra_args"] = [str(arg) for arg in extra_args]
|
||||
|
||||
# 0 / negative means "unset"; the loader reads absence as the app default.
|
||||
for key in ("max_seq_length", "custom_context_length"):
|
||||
parsed = _bounded_int(payload.get(key), minimum = 1, maximum = MAX_SEQ_LENGTH_CEILING)
|
||||
if parsed:
|
||||
entry[key] = parsed
|
||||
|
||||
kv_cache_dtype = _clean_str(payload.get("kv_cache_dtype"), VALID_KV_CACHE_DTYPES)
|
||||
if kv_cache_dtype:
|
||||
entry["kv_cache_dtype"] = kv_cache_dtype
|
||||
|
||||
speculative_type = _clean_str(payload.get("speculative_type"), VALID_SPECULATIVE_TYPES)
|
||||
if speculative_type:
|
||||
entry["speculative_type"] = speculative_type
|
||||
# Only meaningful for the MTP modes; storing it otherwise would resurface
|
||||
# in the UI as an edit the loader silently ignores.
|
||||
if speculative_type in MTP_SPECULATIVE_TYPES:
|
||||
spec_draft_n_max = _bounded_int(
|
||||
payload.get("spec_draft_n_max"), minimum = 1, maximum = 16
|
||||
)
|
||||
if spec_draft_n_max:
|
||||
entry["spec_draft_n_max"] = spec_draft_n_max
|
||||
|
||||
if _coerce_bool(payload.get("tensor_parallel")):
|
||||
entry["tensor_parallel"] = True
|
||||
|
||||
template = payload.get("chat_template_override")
|
||||
if isinstance(template, str) and template.strip():
|
||||
if len(template.encode("utf-8")) <= MAX_CHAT_TEMPLATE_OVERRIDE_BYTES:
|
||||
entry["chat_template_override"] = template
|
||||
|
||||
# Only "manual" is a real override: persisting "auto" would pin the model and
|
||||
# stop it following later changes to the global GPU memory preference.
|
||||
if _clean_str(payload.get("gpu_memory_mode"), VALID_GPU_MEMORY_MODES) == "manual":
|
||||
entry["gpu_memory_mode"] = "manual"
|
||||
|
||||
# -1 is Auto (llama.cpp --fit owns layer sizing), which is also the default,
|
||||
# so only a pinned count >= 0 is worth storing.
|
||||
gpu_layers = _bounded_int(payload.get("gpu_layers"), minimum = 0, maximum = 1024)
|
||||
if gpu_layers is not None:
|
||||
entry["gpu_layers"] = gpu_layers
|
||||
|
||||
n_cpu_moe = _bounded_int(payload.get("n_cpu_moe"), minimum = 1, maximum = 1024)
|
||||
if n_cpu_moe:
|
||||
entry["n_cpu_moe"] = n_cpu_moe
|
||||
|
||||
gpu_ids = payload.get("gpu_ids")
|
||||
if isinstance(gpu_ids, (list, tuple)) and gpu_ids:
|
||||
# De-duplicate, preserving order: resolve_requested_gpu_ids rejects a
|
||||
# repeated id outright, so storing [0, 0] would make every later API load
|
||||
# of this model fail with a 400 that the picker never hits.
|
||||
cleaned_ids: list[int] = []
|
||||
for gid in gpu_ids:
|
||||
parsed = _bounded_int(gid, minimum = 0, maximum = 1024)
|
||||
if parsed is not None and parsed not in cleaned_ids:
|
||||
cleaned_ids.append(parsed)
|
||||
if cleaned_ids:
|
||||
entry["gpu_ids"] = cleaned_ids
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
def resolve_fit_max_seq_length(override: dict[str, Any], *, is_gguf: bool) -> Optional[int]:
|
||||
"""The ``max_seq_length`` an API load should send for this override.
|
||||
|
||||
Mirrors resolveFitMaxSeqLength in the UI (features/chat/presets/preset-policy.ts):
|
||||
under Manual GPU memory with Auto layers, llama.cpp's ``--fit`` owns context
|
||||
sizing, so the load sends the explicit context pin (or 0 to hand sizing over)
|
||||
rather than the stored max sequence length. Returns None to leave the field
|
||||
at the loader's default.
|
||||
"""
|
||||
manual_auto_layers = (
|
||||
is_gguf
|
||||
and override.get("gpu_memory_mode") == "manual"
|
||||
and override.get("gpu_layers") is None
|
||||
)
|
||||
if manual_auto_layers:
|
||||
return override.get("custom_context_length") or 0
|
||||
# max_seq_length wins where both are set. The UI only ever sends it for a
|
||||
# non-GGUF model (a GGUF's context is `custom_context_length`), so in
|
||||
# practice the two never collide from that path; a hand-written or legacy
|
||||
# entry that sets it on a GGUF is honoured, which is this API's contract.
|
||||
return override.get("max_seq_length") or override.get("custom_context_length")
|
||||
|
||||
|
||||
def model_override_load_kwargs(override: dict[str, Any], *, is_gguf: bool) -> dict[str, Any]:
|
||||
"""Map a stored per-model config onto ``LoadRequest`` keyword arguments.
|
||||
|
||||
Mirrors the UI's load payload (features/chat/api/chat-adapter.ts) so an API
|
||||
auto-switch load and a picker load of the same model produce the same command
|
||||
line. GPU placement is GGUF-only there, so it is gated the same way here: a
|
||||
safetensors model loads through HF auto-placement and must not inherit a
|
||||
hidden GGUF GPU pin.
|
||||
"""
|
||||
if not override:
|
||||
return {}
|
||||
kwargs: dict[str, Any] = {}
|
||||
|
||||
max_seq_length = resolve_fit_max_seq_length(override, is_gguf = is_gguf)
|
||||
if max_seq_length is not None:
|
||||
kwargs["max_seq_length"] = max_seq_length
|
||||
for source, target in (
|
||||
("llama_extra_args", "llama_extra_args"),
|
||||
("kv_cache_dtype", "cache_type_kv"),
|
||||
("speculative_type", "speculative_type"),
|
||||
("spec_draft_n_max", "spec_draft_n_max"),
|
||||
("tensor_parallel", "tensor_parallel"),
|
||||
("chat_template_override", "chat_template_override"),
|
||||
):
|
||||
if override.get(source) is not None:
|
||||
kwargs[target] = override[source]
|
||||
|
||||
if is_gguf:
|
||||
if override.get("gpu_memory_mode") is not None:
|
||||
kwargs["gpu_memory_mode"] = override["gpu_memory_mode"]
|
||||
if override.get("gpu_layers") is not None:
|
||||
kwargs["gpu_layers"] = override["gpu_layers"]
|
||||
if override.get("n_cpu_moe") is not None:
|
||||
kwargs["n_cpu_moe"] = override["n_cpu_moe"]
|
||||
if override.get("gpu_ids") is not None:
|
||||
kwargs["gpu_ids"] = override["gpu_ids"]
|
||||
return kwargs
|
||||
|
||||
|
||||
def get_model_overrides() -> dict[str, dict]:
|
||||
"""Per-model launch overrides keyed by model id ({llama_extra_args, max_seq_length})."""
|
||||
"""Per-model launch configs keyed by model id (see normalize_model_override)."""
|
||||
raw = _cached_setting(MODEL_OVERRIDES_SETTING_KEY, None)
|
||||
return raw if isinstance(raw, dict) else {}
|
||||
|
||||
|
|
@ -226,15 +429,22 @@ def set_model_override(
|
|||
model_id: str,
|
||||
llama_extra_args: Optional[list[str]] = None,
|
||||
max_seq_length: Optional[int] = None,
|
||||
**config: Any,
|
||||
) -> dict:
|
||||
"""Upsert one model's launch override; an override with no fields removes it."""
|
||||
"""Upsert one model's launch config; a config with no usable fields removes it.
|
||||
|
||||
The two legacy parameters stay positional for existing callers; every other
|
||||
per-model field is passed by keyword and normalized together.
|
||||
"""
|
||||
if not model_id or not model_id.strip():
|
||||
raise ValueError("model_id is required.")
|
||||
entry: dict[str, Any] = {}
|
||||
if llama_extra_args:
|
||||
entry["llama_extra_args"] = [str(arg) for arg in llama_extra_args]
|
||||
if max_seq_length:
|
||||
entry["max_seq_length"] = max(0, int(max_seq_length))
|
||||
entry = normalize_model_override(
|
||||
{
|
||||
**config,
|
||||
"llama_extra_args": llama_extra_args,
|
||||
"max_seq_length": max_seq_length,
|
||||
}
|
||||
)
|
||||
|
||||
from storage.studio_db import upsert_app_setting_map_entry
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button";
|
|||
import { MascotImg } from "@/components/mascot-img";
|
||||
import { useT } from "@/i18n";
|
||||
import { Route as rootRoute } from "./routes/__root";
|
||||
import { Route as apiMonitorRoute } from "./routes/api";
|
||||
import { Route as dataRecipesRoute } from "./routes/data-recipes";
|
||||
import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId";
|
||||
import { Route as chatRoute } from "./routes/chat";
|
||||
|
|
@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([
|
|||
exportRoute,
|
||||
dataRecipesRoute,
|
||||
dataRecipeRoute,
|
||||
apiMonitorRoute,
|
||||
]);
|
||||
|
||||
function DefaultNotFound() {
|
||||
|
|
|
|||
|
|
@ -5,16 +5,14 @@ import { AppSidebar } from "@/components/app-sidebar";
|
|||
import { Navbar } from "@/components/navbar";
|
||||
import { fetchDeviceType, usePlatformStore } from "@/config/env";
|
||||
import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar";
|
||||
import {
|
||||
SettingsDialog,
|
||||
useSettingsDialogStore,
|
||||
} from "@/features/settings";
|
||||
import { SettingsDialog, useSettingsDialogStore } from "@/features/settings";
|
||||
import {
|
||||
ChatPage,
|
||||
clearNewChatDraft,
|
||||
useChatRuntimeStore,
|
||||
type ChatSearch,
|
||||
} from "@/features/chat";
|
||||
import { ApiMonitorOverlay } from "@/features/api-monitor/api-monitor-overlay";
|
||||
import { RemoteCodeConsentDialog } from "@/features/security";
|
||||
import { HfTokenWarningDialog } from "@/features/hf-auth";
|
||||
import { TransformersUpgradeDialog } from "@/features/transformers-upgrade";
|
||||
|
|
@ -33,13 +31,7 @@ import {
|
|||
useRouterState,
|
||||
} from "@tanstack/react-router";
|
||||
import { AnimatePresence, motion } from "motion/react";
|
||||
import {
|
||||
Suspense,
|
||||
useEffect,
|
||||
useLayoutEffect,
|
||||
useMemo,
|
||||
useState,
|
||||
} from "react";
|
||||
import { Suspense, useEffect, useLayoutEffect, useMemo, useState } from "react";
|
||||
import { AppProvider } from "../provider";
|
||||
|
||||
declare module "@tanstack/react-router" {
|
||||
|
|
@ -76,11 +68,17 @@ const CHAT_ONLY_ALLOWED = new Set([
|
|||
// Export stays reachable on chat-only hosts so the page can show its own grayed-out reason
|
||||
// instead of a silent redirect; it self-gates via export capability, so nothing runs.
|
||||
"/export",
|
||||
// Chat-only hosts (Intel Macs, Apple Silicon without MLX, no-GPU boxes) serve
|
||||
// the OpenAI-compatible API exactly like any other host, so the monitor has to
|
||||
// be reachable there. Without this the floating panel's own "Expand" button
|
||||
// and the Settings > API card both redirect to /chat.
|
||||
"/api-monitor",
|
||||
]);
|
||||
|
||||
function isChatOnlyAllowed(pathname: string): boolean {
|
||||
if (CHAT_ONLY_ALLOWED.has(pathname)) return true;
|
||||
if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/")) return true;
|
||||
if (pathname === "/data-recipes" || pathname.startsWith("/data-recipes/"))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
|
|
@ -224,6 +222,8 @@ function RootLayout() {
|
|||
<AppProvider>
|
||||
<PersonalizationSyncMount />
|
||||
{!isAuthFlowRoute && <SettingsDialog />}
|
||||
{/* Opens itself when API traffic arrives; hides on the full monitor page. */}
|
||||
{!isAuthFlowRoute && <ApiMonitorOverlay />}
|
||||
<HfTokenWarningDialog />
|
||||
<RemoteCodeConsentDialog />
|
||||
<TransformersUpgradeDialog />
|
||||
|
|
@ -241,7 +241,9 @@ function RootLayout() {
|
|||
className="!min-h-0 h-[calc(100dvh-var(--studio-titlebar-height,0px))] overflow-hidden"
|
||||
>
|
||||
<AppSidebar />
|
||||
<SidebarInset className={isChatRoute ? "overflow-hidden" : "overflow-y-auto"}>
|
||||
<SidebarInset
|
||||
className={isChatRoute ? "overflow-hidden" : "overflow-y-auto"}
|
||||
>
|
||||
<Navbar />
|
||||
<div
|
||||
className={`relative flex min-h-0 min-w-0 flex-1 basis-0 flex-col ${isChatRoute ? "overflow-hidden" : "overflow-visible"} ${isChatRoute ? "" : "pt-14 md:pt-[var(--studio-non-chat-content-top-inset,var(--studio-content-top-inset,0px))] md:[--studio-titlebar-height:var(--studio-non-chat-content-top-inset,var(--studio-content-top-inset,0px))]"}`}
|
||||
|
|
|
|||
22
studio/frontend/src/app/routes/api.tsx
Normal file
22
studio/frontend/src/app/routes/api.tsx
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { createRoute, lazyRouteComponent } from "@tanstack/react-router";
|
||||
import { requireAuth } from "../auth-guards";
|
||||
import { Route as rootRoute } from "./__root";
|
||||
|
||||
const ApiMonitorPage = lazyRouteComponent(
|
||||
() => import("@/features/api-monitor"),
|
||||
"ApiMonitorPage",
|
||||
);
|
||||
|
||||
export const Route = createRoute({
|
||||
getParentRoute: () => rootRoute,
|
||||
// Not "/api": the backend owns that prefix (and "/v1"), and its SPA fallback
|
||||
// deliberately 404s those paths so API clients get an API-shaped error rather
|
||||
// than an HTML page. A deep link to /api would never reach the router.
|
||||
path: "/api-monitor",
|
||||
staticData: { title: "API" },
|
||||
beforeLoad: () => requireAuth(),
|
||||
component: ApiMonitorPage,
|
||||
});
|
||||
384
studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx
Normal file
384
studio/frontend/src/features/api-monitor/api-monitor-overlay.tsx
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Floating API monitor: opens itself when API traffic arrives, summarises it
|
||||
// without taking over the window, and links through to the full page.
|
||||
|
||||
import { getApiMonitor } from "@/features/chat/api/chat-api";
|
||||
import type { ApiMonitorEntry } from "@/features/chat/types/api";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
ArrowExpand01Icon,
|
||||
DragDropVerticalIcon,
|
||||
Globe02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { useNavigate, useRouterState } from "@tanstack/react-router";
|
||||
import { XIcon } from "lucide-react";
|
||||
import { AnimatePresence, motion, useDragControls } from "motion/react";
|
||||
import {
|
||||
type PointerEvent,
|
||||
type ReactElement,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { useApiMonitorOverlayStore } from "./overlay-store";
|
||||
import { computeStats } from "./use-api-monitor";
|
||||
|
||||
// Live cadence while the panel is on screen.
|
||||
const OPEN_POLL_MS = 1500;
|
||||
// While closed the poll only has to notice that traffic started, so it backs off.
|
||||
const IDLE_POLL_MS = 5000;
|
||||
// Requests shown in the panel; the rest are one click away on the full page.
|
||||
const VISIBLE_ENTRIES = 4;
|
||||
// How long the API must be quiet before a dismissed panel will open itself again.
|
||||
const REARM_QUIET_MS = 60_000;
|
||||
|
||||
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
|
||||
const V1_PREFIX_RE = /^\/v1\//;
|
||||
|
||||
function compactEndpoint(endpoint: string): string {
|
||||
return endpoint
|
||||
.replace(API_INFERENCE_PREFIX_RE, "/api")
|
||||
.replace(V1_PREFIX_RE, "/");
|
||||
}
|
||||
|
||||
function formatDuration(value?: number | null): string {
|
||||
if (value == null) {
|
||||
return "live";
|
||||
}
|
||||
if (value < 1000) {
|
||||
return `${Math.round(value)}ms`;
|
||||
}
|
||||
return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)}s`;
|
||||
}
|
||||
|
||||
function statusDotClass(status: ApiMonitorEntry["status"]): string {
|
||||
switch (status) {
|
||||
case "running":
|
||||
return "bg-blue-500 animate-pulse";
|
||||
case "error":
|
||||
return "bg-red-500";
|
||||
case "cancelled":
|
||||
return "bg-amber-500";
|
||||
default:
|
||||
return "bg-emerald-500";
|
||||
}
|
||||
}
|
||||
|
||||
function StatCell({
|
||||
label,
|
||||
value,
|
||||
tone,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
tone?: "error" | "active";
|
||||
}): ReactElement {
|
||||
return (
|
||||
<div className="flex min-w-0 flex-col items-center gap-0.5 px-1">
|
||||
<span
|
||||
className={cn(
|
||||
"truncate text-ui-15 font-semibold tabular-nums leading-none tracking-[-0.01em]",
|
||||
tone === "error" && "text-red-600 dark:text-red-400",
|
||||
tone === "active" && "text-blue-600 dark:text-blue-400",
|
||||
!tone && "text-nav-fg",
|
||||
)}
|
||||
>
|
||||
{value}
|
||||
</span>
|
||||
{/* Sentence case: Unsloth metric rows read as words, not headers. */}
|
||||
<span className="truncate text-ui-11 tracking-nav text-muted-foreground">
|
||||
{label}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ApiMonitorOverlay(): ReactElement | null {
|
||||
const { isOpen, suppressed, autoOpen, open, close, setAutoOpen } =
|
||||
useApiMonitorOverlayStore();
|
||||
const navigate = useNavigate();
|
||||
const pathname = useRouterState({ select: (s) => s.location.pathname });
|
||||
const onFullPage = pathname === "/api-monitor";
|
||||
|
||||
const [data, setData] = useState<Awaited<
|
||||
ReturnType<typeof getApiMonitor>
|
||||
> | null>(null);
|
||||
|
||||
// One loop for both jobs: panel contents while open, traffic watch while
|
||||
// closed. Stands down on the full page, which polls for itself.
|
||||
useEffect(() => {
|
||||
if (onFullPage) {
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
let timer: number | undefined;
|
||||
const intervalMs = isOpen ? OPEN_POLL_MS : IDLE_POLL_MS;
|
||||
|
||||
function schedule(): void {
|
||||
timer = window.setTimeout(poll, intervalMs);
|
||||
}
|
||||
|
||||
function poll(): void {
|
||||
// A hidden tab has nobody to show the panel to.
|
||||
if (document.hidden) {
|
||||
schedule();
|
||||
return;
|
||||
}
|
||||
getApiMonitor()
|
||||
.then((next) => {
|
||||
if (!cancelled) setData(next);
|
||||
})
|
||||
.catch(() => {
|
||||
// An unreachable server is the full page's story to tell.
|
||||
if (!cancelled) setData(null);
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) schedule();
|
||||
});
|
||||
}
|
||||
|
||||
poll();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer !== undefined) window.clearTimeout(timer);
|
||||
};
|
||||
}, [isOpen, onFullPage]);
|
||||
|
||||
const entries = useMemo(() => data?.entries ?? [], [data]);
|
||||
const stats = useMemo(() => computeStats(entries), [entries]);
|
||||
|
||||
// Ids already seen. A set, not "the newest id": finishing moves an entry to
|
||||
// the front, so the head flips without any new traffic.
|
||||
const seenIdsRef = useRef<Set<string>>(new Set());
|
||||
// Seeded on the first response even when empty, so the first request of a
|
||||
// fresh session is not mistaken for history.
|
||||
const seededRef = useRef(false);
|
||||
const lastNewEntryAtRef = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
if (data == null) {
|
||||
return;
|
||||
}
|
||||
const ids = data.entries.map((entry) => entry.id);
|
||||
if (!seededRef.current) {
|
||||
seededRef.current = true;
|
||||
seenIdsRef.current = new Set(ids);
|
||||
return;
|
||||
}
|
||||
const seen = seenIdsRef.current;
|
||||
const hasNewTraffic = ids.some((id) => !seen.has(id));
|
||||
// Re-seed each poll so the set stays bounded by the server's ring buffer.
|
||||
seenIdsRef.current = new Set(ids);
|
||||
if (!hasNewTraffic) {
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const quietFor = now - lastNewEntryAtRef.current;
|
||||
lastNewEntryAtRef.current = now;
|
||||
if (!autoOpen || isOpen) {
|
||||
return;
|
||||
}
|
||||
// A dismissal holds for that burst and re-arms only once the API goes
|
||||
// quiet, so the next request cannot re-open it a second later.
|
||||
if (suppressed && quietFor < REARM_QUIET_MS) {
|
||||
return;
|
||||
}
|
||||
open();
|
||||
}, [data, autoOpen, suppressed, isOpen, open]);
|
||||
|
||||
// The backlog built up while the poll was stood down is not new traffic.
|
||||
useEffect(() => {
|
||||
if (onFullPage) {
|
||||
seededRef.current = false;
|
||||
}
|
||||
}, [onFullPage]);
|
||||
|
||||
const [constraintsElement, setConstraintsElement] =
|
||||
useState<HTMLDivElement | null>(null);
|
||||
const constraintsRef = useMemo(
|
||||
() => ({ current: constraintsElement }),
|
||||
[constraintsElement],
|
||||
);
|
||||
const dragControls = useDragControls();
|
||||
|
||||
function startDrag(event: PointerEvent<HTMLDivElement>): void {
|
||||
event.preventDefault();
|
||||
dragControls.start(event);
|
||||
}
|
||||
|
||||
const visible = isOpen && !onFullPage;
|
||||
const serverStatus = data?.status ?? "idle";
|
||||
|
||||
return (
|
||||
<AnimatePresence>
|
||||
{visible && (
|
||||
<div
|
||||
ref={setConstraintsElement}
|
||||
className="pointer-events-none fixed inset-0 z-50"
|
||||
>
|
||||
<motion.div
|
||||
drag={true}
|
||||
dragControls={dragControls}
|
||||
dragListener={false}
|
||||
dragConstraints={constraintsRef}
|
||||
dragElastic={0}
|
||||
dragMomentum={false}
|
||||
initial={{ opacity: 0, scale: 0.94 }}
|
||||
animate={{ opacity: 1, scale: 1 }}
|
||||
exit={{ opacity: 0, scale: 0.94 }}
|
||||
/* Panel language borrowed from the sidebar's user menu and the
|
||||
model selector: no hard border, an inset hairline plus a soft
|
||||
drop shadow (menu-soft-surface), a 20px corner, and the heading
|
||||
font throughout. */
|
||||
className="menu-soft-surface pointer-events-auto fixed bottom-4 right-4 flex w-[400px] max-w-[calc(100vw-2rem)] cursor-default select-none resize flex-col overflow-hidden rounded-[20px] border-0 p-2.5 font-heading ring-0"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2 px-1.5 pb-2 pt-0.5">
|
||||
<div className="flex min-w-0 flex-1 items-center gap-2">
|
||||
<HugeiconsIcon
|
||||
icon={Globe02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-icon shrink-0 text-nav-fg"
|
||||
/>
|
||||
<span className="truncate text-ui-13p5 font-semibold tracking-[0.025em] text-nav-fg dark:tracking-[0.04em]">
|
||||
API monitor
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
"size-1.5 shrink-0 rounded-full",
|
||||
serverStatus === "generating"
|
||||
? "animate-pulse bg-blue-500"
|
||||
: serverStatus === "ready"
|
||||
? "bg-emerald-500"
|
||||
: "bg-muted-foreground",
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-0.5">
|
||||
<div
|
||||
onPointerDown={startDrag}
|
||||
className="flex size-7 touch-none cursor-grab items-center justify-center rounded-full text-muted-foreground transition-colors hover:bg-nav-surface-hover hover:text-foreground active:cursor-grabbing"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={DragDropVerticalIcon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={close}
|
||||
title="Close"
|
||||
aria-label="Close API monitor"
|
||||
className="flex size-7 items-center justify-center rounded-full text-muted-foreground transition-colors hover:bg-nav-surface-hover hover:text-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
>
|
||||
<XIcon className="size-3.5" strokeWidth={1.75} />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p className="truncate px-1.5 pb-2.5 text-ui-11p5 tracking-nav text-muted-foreground">
|
||||
{data?.active_model ?? "No model loaded"}
|
||||
</p>
|
||||
|
||||
{/* Metrics on a soft tile, as the Hub and Train pages group readouts. */}
|
||||
<div className="grid grid-cols-4 rounded-[14px] bg-muted/45 py-2.5 dark:bg-background/45">
|
||||
<StatCell
|
||||
label="Live"
|
||||
value={stats.active.toLocaleString()}
|
||||
tone={stats.active > 0 ? "active" : undefined}
|
||||
/>
|
||||
<StatCell label="Requests" value={stats.total.toLocaleString()} />
|
||||
<StatCell
|
||||
label="Errors"
|
||||
value={stats.errors.toLocaleString()}
|
||||
tone={stats.errors > 0 ? "error" : undefined}
|
||||
/>
|
||||
<StatCell
|
||||
label="Avg"
|
||||
value={
|
||||
stats.avgDurationMs == null
|
||||
? "--"
|
||||
: formatDuration(stats.avgDurationMs)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Borderless rows on 12px hover pills, as in the sidebar. */}
|
||||
<div className="flex flex-col pb-1 pt-1.5">
|
||||
{entries.length === 0 ? (
|
||||
<p className="px-1.5 py-4 text-center text-ui-11p5 tracking-nav text-muted-foreground">
|
||||
No requests yet.
|
||||
</p>
|
||||
) : (
|
||||
entries.slice(0, VISIBLE_ENTRIES).map((entry) => (
|
||||
<div
|
||||
key={entry.id}
|
||||
className="flex h-9 min-w-0 items-center gap-2.5 rounded-[12px] px-3 transition-colors hover:bg-nav-surface-hover"
|
||||
>
|
||||
<span
|
||||
className={cn(
|
||||
"size-1.5 shrink-0 rounded-full",
|
||||
statusDotClass(entry.status),
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
<span className="shrink-0 truncate text-ui-12p5 font-medium tracking-nav text-nav-fg">
|
||||
{compactEndpoint(entry.endpoint)}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
"min-w-0 flex-1 truncate text-ui-11p5 tracking-nav",
|
||||
entry.error
|
||||
? "text-red-600 dark:text-red-400"
|
||||
: "text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
{entry.error ? entry.error : entry.model}
|
||||
</span>
|
||||
<span className="shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{formatDuration(entry.duration_ms)}
|
||||
</span>
|
||||
</div>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Through to payloads, filters and per request tokens. */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
close();
|
||||
void navigate({ to: "/api-monitor" });
|
||||
}}
|
||||
className="mt-1 flex h-[33px] w-full items-center justify-center gap-[8.5px] rounded-full bg-muted/60 text-ui-13p5 font-medium tracking-nav text-nav-fg transition-colors hover:bg-nav-surface-hover focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring dark:bg-background/50"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={ArrowExpand01Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-icon shrink-0"
|
||||
/>
|
||||
Expand to full monitor
|
||||
</button>
|
||||
|
||||
{/* Closing only silences this burst; this is the permanent off. */}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
setAutoOpen(false);
|
||||
close();
|
||||
}}
|
||||
className="mt-1.5 w-full rounded-[12px] py-1 text-ui-11 tracking-nav text-muted-foreground transition-colors hover:text-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
>
|
||||
Stop opening this automatically
|
||||
</button>
|
||||
</motion.div>
|
||||
</div>
|
||||
)}
|
||||
</AnimatePresence>
|
||||
);
|
||||
}
|
||||
778
studio/frontend/src/features/api-monitor/api-monitor-page.tsx
Normal file
778
studio/frontend/src/features/api-monitor/api-monitor-page.tsx
Normal file
|
|
@ -0,0 +1,778 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Full-page monitor for Unsloth's OpenAI-compatible API server.
|
||||
//
|
||||
// This replaces the small console that used to be buried in the API settings
|
||||
// tab. Settings still owns configuration (keys, auto-switch, examples); this
|
||||
// page owns observability -- what is being served right now, which requests
|
||||
// failed and why, and which saved settings a remote load will apply.
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import type { ApiMonitorEntry } from "@/features/chat/types/api";
|
||||
import { useSettingsDialogStore } from "@/features/settings";
|
||||
import { copyToClipboard } from "@/lib/copy-to-clipboard";
|
||||
import { Tick02Icon } from "@/lib/tick-icon";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
Copy01Icon,
|
||||
Delete02Icon,
|
||||
Globe02Icon,
|
||||
PauseIcon,
|
||||
PlayIcon,
|
||||
RefreshIcon,
|
||||
Settings02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { type ReactElement, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { SavedModelSettingsPanel } from "./components/saved-model-settings";
|
||||
import {
|
||||
type MonitorStatusFilter,
|
||||
filterEntries,
|
||||
useApiMonitor,
|
||||
} from "./use-api-monitor";
|
||||
|
||||
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
|
||||
const V1_PREFIX_RE = /^\/v1\//;
|
||||
|
||||
const STATUS_FILTERS: { value: MonitorStatusFilter; label: string }[] = [
|
||||
{ value: "all", label: "All requests" },
|
||||
{ value: "running", label: "In flight" },
|
||||
{ value: "completed", label: "Completed" },
|
||||
{ value: "error", label: "Errors" },
|
||||
{ value: "cancelled", label: "Cancelled" },
|
||||
];
|
||||
|
||||
function formatTime(value: number): string {
|
||||
return new Date(value * 1000).toLocaleTimeString([], {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
}
|
||||
|
||||
function formatDuration(value?: number | null): string {
|
||||
if (value == null) {
|
||||
return "running";
|
||||
}
|
||||
if (value < 1000) {
|
||||
return `${Math.round(value)} ms`;
|
||||
}
|
||||
return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`;
|
||||
}
|
||||
|
||||
function formatCount(value: number): string {
|
||||
return value.toLocaleString();
|
||||
}
|
||||
|
||||
function compactEndpoint(endpoint: string): string {
|
||||
return endpoint
|
||||
.replace(API_INFERENCE_PREFIX_RE, "/api")
|
||||
.replace(V1_PREFIX_RE, "/");
|
||||
}
|
||||
|
||||
function statusDotClass(status: ApiMonitorEntry["status"]): string {
|
||||
switch (status) {
|
||||
case "running":
|
||||
return "bg-blue-500 animate-pulse";
|
||||
case "error":
|
||||
return "bg-red-500";
|
||||
case "cancelled":
|
||||
return "bg-amber-500";
|
||||
default:
|
||||
return "bg-emerald-500";
|
||||
}
|
||||
}
|
||||
|
||||
function statusTextClass(status: ApiMonitorEntry["status"]): string {
|
||||
switch (status) {
|
||||
case "running":
|
||||
return "text-blue-600 dark:text-blue-400";
|
||||
case "error":
|
||||
return "text-red-600 dark:text-red-400";
|
||||
case "cancelled":
|
||||
return "text-amber-600 dark:text-amber-500";
|
||||
default:
|
||||
return "text-emerald-600 dark:text-emerald-500";
|
||||
}
|
||||
}
|
||||
|
||||
function StatCard({
|
||||
label,
|
||||
value,
|
||||
hint,
|
||||
tone,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
hint?: string;
|
||||
tone?: "default" | "error" | "active";
|
||||
}): ReactElement {
|
||||
return (
|
||||
<div className="flex min-w-0 flex-col gap-1 rounded-xl border border-border/60 bg-card px-4 py-3">
|
||||
<span className="truncate text-ui-11 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
{label}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
"truncate text-ui-22 font-semibold leading-tight tracking-[-0.02em]",
|
||||
tone === "error" && "text-red-600 dark:text-red-400",
|
||||
tone === "active" && "text-blue-600 dark:text-blue-400",
|
||||
)}
|
||||
>
|
||||
{value}
|
||||
</span>
|
||||
{hint ? (
|
||||
<span className="truncate text-ui-11 text-muted-foreground">
|
||||
{hint}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function CopyButton({
|
||||
value,
|
||||
label,
|
||||
}: {
|
||||
value: string;
|
||||
label: string;
|
||||
}): ReactElement {
|
||||
const [copied, setCopied] = useState(false);
|
||||
// Clearing the tick on a timer would set state after unmount if the user
|
||||
// navigates away mid-flash, so the timer is cancelled on cleanup.
|
||||
const timerRef = useRef<number | undefined>(undefined);
|
||||
useEffect(
|
||||
() => () => {
|
||||
if (timerRef.current !== undefined) window.clearTimeout(timerRef.current);
|
||||
},
|
||||
[],
|
||||
);
|
||||
return (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
aria-label={label}
|
||||
onClick={async () => {
|
||||
if (await copyToClipboard(value)) {
|
||||
setCopied(true);
|
||||
if (timerRef.current !== undefined) {
|
||||
window.clearTimeout(timerRef.current);
|
||||
}
|
||||
timerRef.current = window.setTimeout(() => setCopied(false), 1800);
|
||||
}
|
||||
}}
|
||||
className="h-7 shrink-0 gap-1.5 px-2 text-ui-11"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={copied ? Tick02Icon : Copy01Icon}
|
||||
strokeWidth={1.75}
|
||||
className={cn(
|
||||
"size-3.5",
|
||||
copied && "text-emerald-600 dark:text-emerald-500",
|
||||
)}
|
||||
/>
|
||||
{copied ? "Copied" : "Copy"}
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
function ContextUsageBar({
|
||||
value,
|
||||
}: { value?: number | null }): ReactElement | null {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
const pct = Math.max(0, Math.min(100, Math.round(value * 100)));
|
||||
return (
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-muted">
|
||||
<div
|
||||
className={cn(
|
||||
"h-full rounded-full transition-[width]",
|
||||
// Near-full context is the usual cause of truncated replies, so it
|
||||
// reads as a warning before it becomes a bug report.
|
||||
pct >= 90
|
||||
? "bg-red-500"
|
||||
: pct >= 75
|
||||
? "bg-amber-500"
|
||||
: "bg-control-accent",
|
||||
)}
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
<span className="shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{pct}%
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function RequestRow({
|
||||
entry,
|
||||
selected,
|
||||
onSelect,
|
||||
}: {
|
||||
entry: ApiMonitorEntry;
|
||||
selected: boolean;
|
||||
onSelect: () => void;
|
||||
}): ReactElement {
|
||||
const preview =
|
||||
entry.error ||
|
||||
entry.reply_preview ||
|
||||
entry.prompt_preview ||
|
||||
(entry.status === "running" ? "Waiting for output…" : "No preview");
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={onSelect}
|
||||
aria-current={selected}
|
||||
className={cn(
|
||||
"flex w-full min-w-0 flex-col gap-1 border-b border-border/50 px-4 py-3 text-left transition-colors last:border-b-0",
|
||||
selected ? "bg-muted/70" : "hover:bg-muted/40",
|
||||
)}
|
||||
>
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<span
|
||||
className={cn(
|
||||
"size-2 shrink-0 rounded-full",
|
||||
statusDotClass(entry.status),
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
<span className="truncate text-ui-13 font-medium text-foreground">
|
||||
{compactEndpoint(entry.endpoint)}
|
||||
</span>
|
||||
<span className="ml-auto shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{formatDuration(entry.duration_ms)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 items-center gap-2 pl-4">
|
||||
<span className="truncate text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</span>
|
||||
<span className="ml-auto shrink-0 text-ui-11 tabular-nums text-muted-foreground">
|
||||
{formatTime(entry.started_at)}
|
||||
</span>
|
||||
</div>
|
||||
<p
|
||||
className={cn(
|
||||
"line-clamp-2 pl-4 text-ui-11 leading-[1.45]",
|
||||
entry.error
|
||||
? "text-red-600 dark:text-red-400"
|
||||
: "text-muted-foreground",
|
||||
)}
|
||||
>
|
||||
{preview}
|
||||
</p>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
function PayloadBlock({
|
||||
title,
|
||||
body,
|
||||
truncated,
|
||||
loading,
|
||||
tone,
|
||||
}: {
|
||||
title: string;
|
||||
body: string;
|
||||
truncated?: boolean;
|
||||
loading?: boolean;
|
||||
tone?: "error";
|
||||
}): ReactElement {
|
||||
return (
|
||||
<section className="flex min-w-0 flex-col gap-1.5">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<h3 className="text-ui-11 font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
{title}
|
||||
</h3>
|
||||
<div className="flex items-center gap-1">
|
||||
{truncated ? (
|
||||
<span className="text-ui-10 text-muted-foreground">
|
||||
preview only
|
||||
</span>
|
||||
) : null}
|
||||
{body ? (
|
||||
<CopyButton value={body} label={`Copy ${title.toLowerCase()}`} />
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
<pre
|
||||
className={cn(
|
||||
"max-h-72 overflow-auto whitespace-pre-wrap break-words rounded-lg bg-muted/50 p-3 text-ui-11 leading-[1.55]",
|
||||
tone === "error" && "bg-red-500/5 text-red-700 dark:text-red-400",
|
||||
)}
|
||||
>
|
||||
{loading && !body ? "Loading…" : body || "–"}
|
||||
</pre>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function RequestDetail({
|
||||
entry,
|
||||
detail,
|
||||
loading,
|
||||
}: {
|
||||
entry: ApiMonitorEntry;
|
||||
detail?: ApiMonitorEntry;
|
||||
loading: boolean;
|
||||
}): ReactElement {
|
||||
// The detail fetch is a separate request, so it can describe an older state of
|
||||
// a still-streaming entry. Prefer it only once it is at least as fresh as the
|
||||
// list row, otherwise the panel would appear to rewind while tokens arrive.
|
||||
const detailIsCurrent =
|
||||
detail != null &&
|
||||
detail.status === entry.status &&
|
||||
detail.updated_at >= entry.updated_at;
|
||||
const prompt = detail?.prompt ?? entry.prompt_preview;
|
||||
const reply = detailIsCurrent
|
||||
? (detail.reply ?? entry.reply_preview)
|
||||
: entry.reply_preview;
|
||||
|
||||
return (
|
||||
<div className="flex min-w-0 flex-col gap-5 p-5">
|
||||
<header className="flex min-w-0 flex-col gap-2">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<span
|
||||
className={cn(
|
||||
"size-2 shrink-0 rounded-full",
|
||||
statusDotClass(entry.status),
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
<span
|
||||
className={cn(
|
||||
"text-ui-11 font-semibold uppercase tracking-wider",
|
||||
statusTextClass(entry.status),
|
||||
)}
|
||||
>
|
||||
{entry.status}
|
||||
</span>
|
||||
<span className="ml-auto shrink-0 font-mono text-ui-10 text-muted-foreground">
|
||||
{entry.id}
|
||||
</span>
|
||||
</div>
|
||||
<h2 className="min-w-0 break-all font-mono text-ui-13 font-medium text-foreground">
|
||||
{entry.method} {entry.endpoint}
|
||||
</h2>
|
||||
<p className="min-w-0 break-all text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<dl className="grid grid-cols-2 gap-x-4 gap-y-3 rounded-xl border border-border/60 bg-card px-4 py-3 sm:grid-cols-3">
|
||||
{[
|
||||
{ label: "Started", value: formatTime(entry.started_at) },
|
||||
{ label: "Duration", value: formatDuration(entry.duration_ms) },
|
||||
{
|
||||
label: "Prompt tokens",
|
||||
value:
|
||||
entry.prompt_tokens != null
|
||||
? formatCount(entry.prompt_tokens)
|
||||
: "–",
|
||||
},
|
||||
{
|
||||
label: "Completion tokens",
|
||||
value:
|
||||
entry.completion_tokens != null
|
||||
? formatCount(entry.completion_tokens)
|
||||
: "–",
|
||||
},
|
||||
{
|
||||
label: "Total tokens",
|
||||
value:
|
||||
entry.total_tokens != null
|
||||
? formatCount(entry.total_tokens)
|
||||
: "–",
|
||||
},
|
||||
{
|
||||
label: "Context",
|
||||
value:
|
||||
entry.context_length != null
|
||||
? formatCount(entry.context_length)
|
||||
: "–",
|
||||
},
|
||||
].map((item) => (
|
||||
<div key={item.label} className="flex min-w-0 flex-col gap-0.5">
|
||||
<dt className="truncate text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
{item.label}
|
||||
</dt>
|
||||
<dd className="truncate text-ui-13 tabular-nums text-foreground">
|
||||
{item.value}
|
||||
</dd>
|
||||
</div>
|
||||
))}
|
||||
</dl>
|
||||
|
||||
{entry.context_usage != null ? (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<span className="text-ui-11 font-semibold uppercase tracking-wider text-muted-foreground">
|
||||
Context used
|
||||
</span>
|
||||
<ContextUsageBar value={entry.context_usage} />
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{entry.error ? (
|
||||
<PayloadBlock title="Error" body={entry.error} tone="error" />
|
||||
) : null}
|
||||
|
||||
<PayloadBlock
|
||||
title="Prompt"
|
||||
body={prompt}
|
||||
truncated={entry.prompt_truncated && detail?.prompt == null}
|
||||
loading={loading}
|
||||
/>
|
||||
<PayloadBlock
|
||||
title="Reply"
|
||||
body={reply}
|
||||
truncated={entry.reply_truncated && !detailIsCurrent}
|
||||
loading={loading}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ApiMonitorPage(): ReactElement {
|
||||
const {
|
||||
data,
|
||||
entries,
|
||||
stats,
|
||||
error,
|
||||
loading,
|
||||
refreshing,
|
||||
paused,
|
||||
setPaused,
|
||||
refresh,
|
||||
clear,
|
||||
details,
|
||||
loadingDetails,
|
||||
requestDetail,
|
||||
} = useApiMonitor();
|
||||
const [statusFilter, setStatusFilter] = useState<MonitorStatusFilter>("all");
|
||||
const [query, setQuery] = useState("");
|
||||
const [selectedId, setSelectedId] = useState<string | null>(null);
|
||||
|
||||
const visible = useMemo(
|
||||
() => filterEntries(entries, statusFilter, query),
|
||||
[entries, statusFilter, query],
|
||||
);
|
||||
const selected = useMemo(
|
||||
() => visible.find((entry) => entry.id === selectedId) ?? null,
|
||||
[visible, selectedId],
|
||||
);
|
||||
|
||||
// Refetch the selected entry while it streams so the payload grows with the
|
||||
// reply. Keyed on identity and revision, never on `details`: the fetch rewrites
|
||||
// `details` on every success, so depending on it loops on the detail endpoint,
|
||||
// which takes the same lock every generated token does.
|
||||
const selectedId_ = selected?.id ?? null;
|
||||
const selectedUpdatedAt = selected?.updated_at ?? null;
|
||||
const selectedIsMissing = selectedId_ != null && details[selectedId_] == null;
|
||||
const lastFetchedRef = useRef<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (selectedId_ == null) {
|
||||
return;
|
||||
}
|
||||
// `updated_at` advances per poll while streaming and settles when terminal.
|
||||
// A missing payload always retries, covering a fetch that failed late.
|
||||
const revision = `${selectedId_}@${selectedUpdatedAt ?? ""}`;
|
||||
if (!selectedIsMissing && lastFetchedRef.current === revision) {
|
||||
return;
|
||||
}
|
||||
lastFetchedRef.current = revision;
|
||||
requestDetail(selectedId_);
|
||||
}, [selectedId_, selectedUpdatedAt, selectedIsMissing, requestDetail]);
|
||||
|
||||
const baseUrl =
|
||||
typeof window === "undefined" ? "" : `${window.location.origin}/v1`;
|
||||
const serverStatus = data?.status ?? "idle";
|
||||
const statusCopy =
|
||||
serverStatus === "generating"
|
||||
? "Serving requests"
|
||||
: serverStatus === "ready"
|
||||
? "Ready"
|
||||
: "No model loaded";
|
||||
|
||||
return (
|
||||
<main className="mx-auto flex w-full max-w-6xl flex-col gap-6 px-6 pb-10 pt-12 font-heading sm:px-10">
|
||||
<header className="flex flex-wrap items-start justify-between gap-4">
|
||||
<div className="flex min-w-0 flex-col gap-1">
|
||||
<h1 className="text-ui-30 font-semibold leading-[1.04] tracking-[-0.028em] text-foreground sm:text-ui-34">
|
||||
API
|
||||
</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Live traffic through Unsloth's OpenAI-compatible server.
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => setPaused(!paused)}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={paused ? PlayIcon : PauseIcon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
{paused ? "Resume" : "Pause"}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={refresh}
|
||||
disabled={refreshing}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={RefreshIcon}
|
||||
strokeWidth={1.75}
|
||||
className={cn("size-4", refreshing && "animate-spin")}
|
||||
/>
|
||||
Refresh
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={entries.length === 0}
|
||||
onClick={() => {
|
||||
setSelectedId(null);
|
||||
void clear();
|
||||
}}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Delete02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
Clear log
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() =>
|
||||
useSettingsDialogStore.getState().openDialog("api-keys")
|
||||
}
|
||||
className="h-9 gap-1.5 rounded-full"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Settings02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
API settings
|
||||
</Button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* Server summary: the two things you check first when a client can't
|
||||
reach the API -- the base URL to point it at, and what is loaded. */}
|
||||
<section className="flex flex-wrap items-center gap-x-6 gap-y-3 rounded-xl border border-border/60 bg-card px-4 py-3">
|
||||
<div className="flex min-w-0 items-center gap-2.5">
|
||||
<span className="flex size-8 shrink-0 items-center justify-center rounded-lg border border-border/60 bg-muted/40">
|
||||
<HugeiconsIcon
|
||||
icon={Globe02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
</span>
|
||||
<div className="flex min-w-0 flex-col">
|
||||
<span className="text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
Base URL
|
||||
</span>
|
||||
<span className="truncate font-mono text-ui-12 text-foreground">
|
||||
{baseUrl}
|
||||
</span>
|
||||
</div>
|
||||
<CopyButton value={baseUrl} label="Copy API base URL" />
|
||||
</div>
|
||||
<div className="flex min-w-0 flex-col">
|
||||
<span className="text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
Status
|
||||
</span>
|
||||
<span className="flex items-center gap-1.5 text-ui-12 text-foreground">
|
||||
<span
|
||||
className={cn(
|
||||
"size-2 rounded-full",
|
||||
serverStatus === "generating"
|
||||
? "bg-blue-500 animate-pulse"
|
||||
: serverStatus === "ready"
|
||||
? "bg-emerald-500"
|
||||
: "bg-muted-foreground",
|
||||
)}
|
||||
aria-hidden={true}
|
||||
/>
|
||||
{statusCopy}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex min-w-0 flex-1 flex-col">
|
||||
<span className="text-ui-10 font-medium uppercase tracking-wider text-muted-foreground">
|
||||
Loaded model
|
||||
</span>
|
||||
<span className="truncate text-ui-12 text-foreground">
|
||||
{data?.active_model ?? "None"}
|
||||
{data?.context_length
|
||||
? ` · ${formatCount(data.context_length)} ctx`
|
||||
: ""}
|
||||
</span>
|
||||
</div>
|
||||
{paused ? (
|
||||
<span className="rounded-full border border-amber-500/40 bg-amber-500/10 px-2.5 py-1 text-ui-11 font-medium text-amber-700 dark:text-amber-500">
|
||||
Paused
|
||||
</span>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
{error ? (
|
||||
<div className="rounded-xl border border-red-500/40 bg-red-500/5 px-4 py-3 text-sm text-red-600 dark:text-red-400">
|
||||
{error}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<section className="grid grid-cols-2 gap-3 sm:grid-cols-3 lg:grid-cols-6">
|
||||
<StatCard
|
||||
label="In flight"
|
||||
value={formatCount(stats.active)}
|
||||
tone={stats.active > 0 ? "active" : "default"}
|
||||
/>
|
||||
<StatCard
|
||||
label="Requests"
|
||||
value={formatCount(stats.total)}
|
||||
hint="recent window"
|
||||
/>
|
||||
<StatCard label="Completed" value={formatCount(stats.completed)} />
|
||||
<StatCard
|
||||
label="Errors"
|
||||
value={formatCount(stats.errors)}
|
||||
tone={stats.errors > 0 ? "error" : "default"}
|
||||
hint={
|
||||
stats.errorRate != null
|
||||
? `${Math.round(stats.errorRate * 100)}% of finished`
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
<StatCard
|
||||
label="Avg latency"
|
||||
value={
|
||||
stats.avgDurationMs == null
|
||||
? "–"
|
||||
: formatDuration(stats.avgDurationMs)
|
||||
}
|
||||
hint={
|
||||
stats.maxDurationMs != null
|
||||
? `max ${formatDuration(stats.maxDurationMs)}`
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
<StatCard
|
||||
label="Throughput"
|
||||
value={
|
||||
stats.tokensPerSecond == null
|
||||
? "–"
|
||||
: `${stats.tokensPerSecond.toFixed(1)} tok/s`
|
||||
}
|
||||
hint={`${formatCount(stats.totalTokens)} tokens`}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section className="flex min-h-0 flex-col overflow-hidden rounded-xl border border-border/60 bg-card">
|
||||
<div className="flex flex-wrap items-center gap-2 border-b border-border/60 px-4 py-3">
|
||||
<Input
|
||||
value={query}
|
||||
onChange={(event) => setQuery(event.target.value)}
|
||||
placeholder="Search model, endpoint, preview or error"
|
||||
aria-label="Search API requests"
|
||||
className="h-9 w-full min-w-0 flex-1 rounded-full border-none bg-muted shadow-none dark:bg-background sm:w-64 sm:flex-none"
|
||||
/>
|
||||
<Select
|
||||
value={statusFilter}
|
||||
onValueChange={(value) =>
|
||||
setStatusFilter(value as MonitorStatusFilter)
|
||||
}
|
||||
>
|
||||
<SelectTrigger
|
||||
aria-label="Filter by status"
|
||||
className="h-9 w-[150px] rounded-full border-none bg-muted shadow-none dark:bg-background"
|
||||
>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{STATUS_FILTERS.map((option) => (
|
||||
<SelectItem key={option.value} value={option.value}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<span className="ml-auto shrink-0 text-ui-11 text-muted-foreground">
|
||||
{formatCount(visible.length)} of {formatCount(entries.length)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="grid min-h-0 grid-cols-1 lg:grid-cols-[minmax(0,380px)_minmax(0,1fr)]">
|
||||
<div className="max-h-[560px] min-h-[220px] overflow-y-auto border-b border-border/60 lg:border-b-0 lg:border-r">
|
||||
{loading ? (
|
||||
<div className="flex flex-col gap-3 p-4">
|
||||
{[0, 1, 2].map((i) => (
|
||||
<Skeleton key={i} className="h-16 w-full rounded-lg" />
|
||||
))}
|
||||
</div>
|
||||
) : visible.length === 0 ? (
|
||||
<p className="px-4 py-10 text-center text-sm text-muted-foreground">
|
||||
{entries.length === 0
|
||||
? "No API traffic yet. Point a client at the base URL above to see requests here."
|
||||
: "No requests match this filter."}
|
||||
</p>
|
||||
) : (
|
||||
visible.map((entry) => (
|
||||
<RequestRow
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
selected={entry.id === selectedId}
|
||||
onSelect={() => setSelectedId(entry.id)}
|
||||
/>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="max-h-[560px] min-h-[220px] overflow-y-auto">
|
||||
{selected ? (
|
||||
<RequestDetail
|
||||
entry={selected}
|
||||
detail={details[selected.id]}
|
||||
loading={loadingDetails.has(selected.id)}
|
||||
/>
|
||||
) : (
|
||||
<p className="flex h-full items-center justify-center px-6 py-10 text-center text-sm text-muted-foreground">
|
||||
Select a request to inspect its prompt, reply, tokens and
|
||||
errors.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<SavedModelSettingsPanel />
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// What a remote load will actually apply, which is otherwise unanswerable from
|
||||
// outside the process.
|
||||
//
|
||||
// Read only on purpose: the config lives both here and in the browser's own
|
||||
// per-model store, and the model's settings page is the only place that owns
|
||||
// both, so it is the only place that can forget a model completely.
|
||||
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import {
|
||||
type ApiModelOverride,
|
||||
type ApiModelOverrides,
|
||||
fetchModelOverrides,
|
||||
} from "@/features/model-picker/api/model-overrides";
|
||||
import { type ReactElement, useCallback, useEffect, useState } from "react";
|
||||
|
||||
/** Human-readable summary of the fields the loader will apply, in load order. */
|
||||
function describeOverride(override: ApiModelOverride): string[] {
|
||||
const parts: string[] = [];
|
||||
if (override.custom_context_length) {
|
||||
parts.push(`${override.custom_context_length.toLocaleString()} context`);
|
||||
}
|
||||
if (override.max_seq_length) {
|
||||
parts.push(`${override.max_seq_length.toLocaleString()} max seq`);
|
||||
}
|
||||
if (override.kv_cache_dtype) {
|
||||
parts.push(`KV ${override.kv_cache_dtype}`);
|
||||
}
|
||||
if (override.speculative_type) {
|
||||
parts.push(
|
||||
override.spec_draft_n_max
|
||||
? `spec ${override.speculative_type} ×${override.spec_draft_n_max}`
|
||||
: `spec ${override.speculative_type}`,
|
||||
);
|
||||
}
|
||||
if (override.tensor_parallel) {
|
||||
parts.push("tensor parallel");
|
||||
}
|
||||
if (override.gpu_memory_mode === "manual") {
|
||||
parts.push("manual GPU memory");
|
||||
}
|
||||
if (override.gpu_layers != null) {
|
||||
parts.push(`${override.gpu_layers} GPU layers`);
|
||||
}
|
||||
if (override.n_cpu_moe) {
|
||||
parts.push(`${override.n_cpu_moe} MoE layers on CPU`);
|
||||
}
|
||||
if (override.gpu_ids?.length) {
|
||||
parts.push(`GPU ${override.gpu_ids.join(", ")}`);
|
||||
}
|
||||
if (override.chat_template_override) {
|
||||
parts.push("custom chat template");
|
||||
}
|
||||
if (override.llama_extra_args?.length) {
|
||||
parts.push(override.llama_extra_args.join(" "));
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
export function SavedModelSettingsPanel(): ReactElement {
|
||||
const [overrides, setOverrides] = useState<ApiModelOverrides | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const load = useCallback(async () => {
|
||||
try {
|
||||
setOverrides(await fetchModelOverrides());
|
||||
setError(null);
|
||||
} catch (err: unknown) {
|
||||
setError(
|
||||
err instanceof Error
|
||||
? err.message
|
||||
: "Could not load saved model settings",
|
||||
);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
const entries = Object.entries(overrides ?? {});
|
||||
|
||||
return (
|
||||
<section className="flex flex-col gap-3">
|
||||
<div className="flex flex-col gap-1">
|
||||
<h2 className="text-ui-16 font-semibold tracking-[-0.01em] text-foreground">
|
||||
Settings applied on API load
|
||||
</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
When a request names one of these models, Unsloth loads it with these
|
||||
settings, the same ones you saved in the model's settings page.
|
||||
Models without an entry load with app defaults. Edit or forget an
|
||||
entry from that model's settings, which keeps this list and the
|
||||
picker in step.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{error ? (
|
||||
<div className="rounded-xl border border-red-500/40 bg-red-500/5 px-4 py-3 text-sm text-red-600 dark:text-red-400">
|
||||
{error}
|
||||
</div>
|
||||
) : overrides == null ? (
|
||||
<div className="flex flex-col gap-2">
|
||||
{[0, 1].map((i) => (
|
||||
<Skeleton key={i} className="h-14 w-full rounded-xl" />
|
||||
))}
|
||||
</div>
|
||||
) : entries.length === 0 ? (
|
||||
<p className="rounded-xl border border-border/60 bg-card px-4 py-6 text-center text-sm text-muted-foreground">
|
||||
No saved model settings yet. Open a model's settings, turn on
|
||||
"Remember for this model", and it will be applied to API
|
||||
loads too.
|
||||
</p>
|
||||
) : (
|
||||
<ul className="flex flex-col gap-2">
|
||||
{entries.map(([modelId, override]) => {
|
||||
const summary = describeOverride(override);
|
||||
return (
|
||||
<li
|
||||
key={modelId}
|
||||
className="flex min-w-0 items-start gap-3 rounded-xl border border-border/60 bg-card px-4 py-3"
|
||||
>
|
||||
<div className="flex min-w-0 flex-1 flex-col gap-1">
|
||||
<span className="min-w-0 break-all font-mono text-ui-12 font-medium text-foreground">
|
||||
{modelId}
|
||||
</span>
|
||||
<span className="min-w-0 break-words text-ui-11 text-muted-foreground">
|
||||
{summary.length > 0 ? summary.join(" · ") : "App defaults"}
|
||||
</span>
|
||||
</div>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
13
studio/frontend/src/features/api-monitor/index.ts
Normal file
13
studio/frontend/src/features/api-monitor/index.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
export { ApiMonitorPage } from "./api-monitor-page";
|
||||
export { ApiMonitorOverlay } from "./api-monitor-overlay";
|
||||
export { useApiMonitorOverlayStore } from "./overlay-store";
|
||||
export {
|
||||
computeStats,
|
||||
filterEntries,
|
||||
useApiMonitor,
|
||||
type MonitorStats,
|
||||
type MonitorStatusFilter,
|
||||
} from "./use-api-monitor";
|
||||
48
studio/frontend/src/features/api-monitor/overlay-store.ts
Normal file
48
studio/frontend/src/features/api-monitor/overlay-store.ts
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { create } from "zustand";
|
||||
import { persist } from "zustand/middleware";
|
||||
|
||||
interface ApiMonitorOverlayState {
|
||||
/** Whether the floating panel is on screen right now. Session state. */
|
||||
isOpen: boolean;
|
||||
/** Set on close so the panel does not pop back during the same burst. */
|
||||
suppressed: boolean;
|
||||
/** Persisted opt out: when false the panel never opens itself. */
|
||||
autoOpen: boolean;
|
||||
open: () => void;
|
||||
close: () => void;
|
||||
setAutoOpen: (autoOpen: boolean) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Only `autoOpen` persists. Open/closed is session state: a dismissal lasts the
|
||||
* sitting, not forever.
|
||||
*/
|
||||
export const useApiMonitorOverlayStore = create<ApiMonitorOverlayState>()(
|
||||
persist(
|
||||
(set) => ({
|
||||
isOpen: false,
|
||||
suppressed: false,
|
||||
autoOpen: true,
|
||||
open: () => set({ isOpen: true, suppressed: false }),
|
||||
close: () => set({ isOpen: false, suppressed: true }),
|
||||
setAutoOpen: (autoOpen) => set({ autoOpen }),
|
||||
}),
|
||||
{
|
||||
name: "unsloth_api_monitor_overlay",
|
||||
version: 1,
|
||||
partialize: (state) => ({ autoOpen: state.autoOpen }),
|
||||
// Explicit merge so an older stored payload cannot resurrect `isOpen`.
|
||||
merge: (persisted, current) => ({
|
||||
...current,
|
||||
autoOpen:
|
||||
typeof (persisted as { autoOpen?: unknown } | null)?.autoOpen ===
|
||||
"boolean"
|
||||
? (persisted as { autoOpen: boolean }).autoOpen
|
||||
: current.autoOpen,
|
||||
}),
|
||||
},
|
||||
),
|
||||
);
|
||||
297
studio/frontend/src/features/api-monitor/use-api-monitor.ts
Normal file
297
studio/frontend/src/features/api-monitor/use-api-monitor.ts
Normal file
|
|
@ -0,0 +1,297 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import {
|
||||
clearApiMonitor,
|
||||
getApiMonitor,
|
||||
getApiMonitorEntry,
|
||||
} from "@/features/chat/api/chat-api";
|
||||
import type {
|
||||
ApiMonitorEntry,
|
||||
ApiMonitorResponse,
|
||||
} from "@/features/chat/types/api";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
/** Poll cadence while the monitor is live. Matches the settings console it replaces. */
|
||||
const POLL_INTERVAL_MS = 1500;
|
||||
|
||||
export type MonitorStatusFilter =
|
||||
| "all"
|
||||
| "running"
|
||||
| "completed"
|
||||
| "error"
|
||||
| "cancelled";
|
||||
|
||||
export interface MonitorStats {
|
||||
active: number;
|
||||
total: number;
|
||||
completed: number;
|
||||
errors: number;
|
||||
cancelled: number;
|
||||
/** Mean duration over finished requests, or null when none have finished. */
|
||||
avgDurationMs: number | null;
|
||||
/** Slowest finished request, for spotting a single pathological call. */
|
||||
maxDurationMs: number | null;
|
||||
totalTokens: number;
|
||||
/** Share of finished requests that failed, 0-1. Null when nothing finished. */
|
||||
errorRate: number | null;
|
||||
/** Mean completion tokens per second over requests that reported both. */
|
||||
tokensPerSecond: number | null;
|
||||
}
|
||||
|
||||
function isTerminal(entry: ApiMonitorEntry): boolean {
|
||||
return entry.status !== "running";
|
||||
}
|
||||
|
||||
function completionTokens(entry: ApiMonitorEntry): number | null {
|
||||
if (entry.completion_tokens != null) {
|
||||
return entry.completion_tokens;
|
||||
}
|
||||
// Some providers only report a total; subtracting the prompt is the best
|
||||
// available estimate of what was actually generated.
|
||||
if (entry.total_tokens != null && entry.prompt_tokens != null) {
|
||||
return Math.max(0, entry.total_tokens - entry.prompt_tokens);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function entryTokens(entry: ApiMonitorEntry): number {
|
||||
if (entry.total_tokens != null) {
|
||||
return entry.total_tokens;
|
||||
}
|
||||
return (entry.prompt_tokens ?? 0) + (entry.completion_tokens ?? 0);
|
||||
}
|
||||
|
||||
export function computeStats(entries: ApiMonitorEntry[]): MonitorStats {
|
||||
let active = 0;
|
||||
let completed = 0;
|
||||
let errors = 0;
|
||||
let cancelled = 0;
|
||||
let totalTokens = 0;
|
||||
let durationSum = 0;
|
||||
let durationCount = 0;
|
||||
let maxDurationMs: number | null = null;
|
||||
// Throughput is aggregated as total tokens over total time, not as the mean of
|
||||
// each request's rate. Averaging rates lets one tiny fast request outweigh a
|
||||
// long slow one, which is the opposite of what someone debugging wants to see.
|
||||
let generatedTokens = 0;
|
||||
let generatedDurationMs = 0;
|
||||
|
||||
for (const entry of entries) {
|
||||
totalTokens += entryTokens(entry);
|
||||
if (entry.status === "running") {
|
||||
active += 1;
|
||||
} else if (entry.status === "error") {
|
||||
errors += 1;
|
||||
} else if (entry.status === "cancelled") {
|
||||
cancelled += 1;
|
||||
} else {
|
||||
completed += 1;
|
||||
}
|
||||
const duration = entry.duration_ms;
|
||||
if (duration != null && isTerminal(entry)) {
|
||||
durationSum += duration;
|
||||
durationCount += 1;
|
||||
maxDurationMs =
|
||||
maxDurationMs == null ? duration : Math.max(maxDurationMs, duration);
|
||||
const generated = completionTokens(entry);
|
||||
// Sub-millisecond durations would divide into a meaningless rate.
|
||||
if (generated != null && generated > 0 && duration > 0) {
|
||||
generatedTokens += generated;
|
||||
generatedDurationMs += duration;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const finished = completed + errors + cancelled;
|
||||
return {
|
||||
active,
|
||||
total: entries.length,
|
||||
completed,
|
||||
errors,
|
||||
cancelled,
|
||||
avgDurationMs: durationCount > 0 ? durationSum / durationCount : null,
|
||||
maxDurationMs,
|
||||
totalTokens,
|
||||
errorRate: finished > 0 ? errors / finished : null,
|
||||
tokensPerSecond:
|
||||
generatedDurationMs > 0
|
||||
? generatedTokens / (generatedDurationMs / 1000)
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function filterEntries(
|
||||
entries: ApiMonitorEntry[],
|
||||
status: MonitorStatusFilter,
|
||||
query: string,
|
||||
): ApiMonitorEntry[] {
|
||||
const needle = query.trim().toLowerCase();
|
||||
return entries.filter((entry) => {
|
||||
if (status !== "all" && entry.status !== status) {
|
||||
return false;
|
||||
}
|
||||
if (!needle) {
|
||||
return true;
|
||||
}
|
||||
// Search the fields a debugging session actually keys off: which model,
|
||||
// which endpoint, and the previews/error text visible in the row.
|
||||
return (
|
||||
entry.model.toLowerCase().includes(needle) ||
|
||||
entry.endpoint.toLowerCase().includes(needle) ||
|
||||
entry.prompt_preview.toLowerCase().includes(needle) ||
|
||||
entry.reply_preview.toLowerCase().includes(needle) ||
|
||||
(entry.error ?? "").toLowerCase().includes(needle)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
interface UseApiMonitorResult {
|
||||
data: ApiMonitorResponse | null;
|
||||
entries: ApiMonitorEntry[];
|
||||
stats: MonitorStats;
|
||||
error: string | null;
|
||||
/** True until the first response lands, so the page can show skeletons once. */
|
||||
loading: boolean;
|
||||
refreshing: boolean;
|
||||
paused: boolean;
|
||||
setPaused: (paused: boolean) => void;
|
||||
refresh: () => void;
|
||||
clear: () => Promise<void>;
|
||||
/** Full prompt/reply for entries the user expanded, keyed by entry id. */
|
||||
details: Record<string, ApiMonitorEntry>;
|
||||
loadingDetails: ReadonlySet<string>;
|
||||
requestDetail: (id: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Live view of the server's OpenAI-compatible API traffic.
|
||||
*
|
||||
* Polls rather than streams because the backing monitor is an in-memory ring
|
||||
* buffer with no change feed. Polling is self-rescheduling (never overlapping),
|
||||
* and pausing stops it entirely so a user reading a stalled request's payload
|
||||
* isn't fighting a list that reorders under them.
|
||||
*
|
||||
* `intervalMs` lets a caller trade freshness for cost: the full page wants the
|
||||
* default live cadence, while the floating overlay slows right down when it is
|
||||
* closed and only watching for the traffic that should pop it open.
|
||||
*/
|
||||
export function useApiMonitor({
|
||||
intervalMs = POLL_INTERVAL_MS,
|
||||
}: { intervalMs?: number } = {}): UseApiMonitorResult {
|
||||
const [data, setData] = useState<ApiMonitorResponse | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [paused, setPaused] = useState(false);
|
||||
const [details, setDetails] = useState<Record<string, ApiMonitorEntry>>({});
|
||||
const [loadingDetails, setLoadingDetails] = useState<Set<string>>(
|
||||
() => new Set(),
|
||||
);
|
||||
// Mirrors `loadingDetails` outside React state so the fetch guard sees writes
|
||||
// from the same tick (state updates are async and would let duplicates through).
|
||||
const inFlightDetails = useRef<Set<string>>(new Set());
|
||||
|
||||
const load = useCallback(async (): Promise<void> => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
const next = await getApiMonitor();
|
||||
setData(next);
|
||||
setError(null);
|
||||
} catch (err: unknown) {
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (paused) {
|
||||
return;
|
||||
}
|
||||
let cancelled = false;
|
||||
let timer: number | undefined;
|
||||
|
||||
function poll(): void {
|
||||
getApiMonitor()
|
||||
.then((next) => {
|
||||
if (cancelled) return;
|
||||
setData(next);
|
||||
setError(null);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (cancelled) return;
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
})
|
||||
.finally(() => {
|
||||
if (cancelled) return;
|
||||
setLoading(false);
|
||||
timer = window.setTimeout(poll, intervalMs);
|
||||
});
|
||||
}
|
||||
|
||||
poll();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer !== undefined) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
}, [paused, intervalMs]);
|
||||
|
||||
const requestDetail = useCallback((id: string): void => {
|
||||
if (inFlightDetails.current.has(id)) {
|
||||
return;
|
||||
}
|
||||
inFlightDetails.current.add(id);
|
||||
setLoadingDetails((prev) => new Set(prev).add(id));
|
||||
getApiMonitorEntry(id)
|
||||
.then((entry) => {
|
||||
setDetails((prev) => ({ ...prev, [id]: entry }));
|
||||
})
|
||||
.catch(() => {
|
||||
// The entry aged out of the ring buffer; drop any stale copy so the UI
|
||||
// falls back to the row previews instead of showing a frozen payload.
|
||||
setDetails((prev) => {
|
||||
if (!(id in prev)) return prev;
|
||||
const next = { ...prev };
|
||||
delete next[id];
|
||||
return next;
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
inFlightDetails.current.delete(id);
|
||||
setLoadingDetails((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(id);
|
||||
return next;
|
||||
});
|
||||
});
|
||||
}, []);
|
||||
|
||||
const clear = useCallback(async (): Promise<void> => {
|
||||
await clearApiMonitor();
|
||||
setDetails({});
|
||||
await load();
|
||||
}, [load]);
|
||||
|
||||
const entries = useMemo(() => data?.entries ?? [], [data]);
|
||||
const stats = useMemo(() => computeStats(entries), [entries]);
|
||||
|
||||
return {
|
||||
data,
|
||||
entries,
|
||||
stats,
|
||||
error,
|
||||
loading,
|
||||
refreshing,
|
||||
paused,
|
||||
setPaused,
|
||||
refresh: () => void load(),
|
||||
clear,
|
||||
details,
|
||||
loadingDetails,
|
||||
requestDetail,
|
||||
};
|
||||
}
|
||||
|
|
@ -129,6 +129,13 @@ export async function getApiMonitorEntry(id: string): Promise<ApiMonitorEntry> {
|
|||
return parseJsonOrThrow<ApiMonitorEntry>(response);
|
||||
}
|
||||
|
||||
export async function clearApiMonitor(): Promise<void> {
|
||||
const response = await authFetch("/api/inference/monitor", {
|
||||
method: "DELETE",
|
||||
});
|
||||
await parseJsonOrThrow<{ cleared: boolean }>(response);
|
||||
}
|
||||
|
||||
export async function loadModel(
|
||||
payload: LoadModelRequest,
|
||||
): Promise<LoadModelResponse> {
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import {
|
|||
ArrowReloadHorizontalIcon,
|
||||
Delete02Icon,
|
||||
Download01Icon,
|
||||
Settings02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import {
|
||||
|
|
@ -82,6 +83,40 @@ export function CardDivider() {
|
|||
);
|
||||
}
|
||||
|
||||
/** Gear that opens a downloaded model's full settings page. */
|
||||
export function CardSettingsButton({
|
||||
label,
|
||||
onClick,
|
||||
}: {
|
||||
label: string;
|
||||
onClick: () => void;
|
||||
}) {
|
||||
return (
|
||||
<Tooltip>
|
||||
<TooltipTrigger asChild>
|
||||
<button
|
||||
type="button"
|
||||
aria-label={label}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
onClick();
|
||||
}}
|
||||
className="inline-flex size-7 shrink-0 cursor-pointer items-center justify-center rounded-full text-muted-foreground opacity-0 transition-[opacity,background-color,color] duration-150 hover:bg-foreground/[0.06] hover:text-foreground focus-visible:opacity-100 group-hover/dl:opacity-100 dark:hover:bg-white/[0.08]"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Settings02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
</button>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top" className="tooltip-compact">
|
||||
{label}
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
export function CardDeleteButton({
|
||||
label,
|
||||
onClick,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,138 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Full-page settings for one model, opened from the Hub.
|
||||
//
|
||||
// The same controls exist in the chat picker's popover, but a popover is a poor
|
||||
// place to work through every knob a model has. This gives them a page, and
|
||||
// states plainly that whatever is saved here is what an API load will use --
|
||||
// the settings are mirrored server-side by ModelConfigPage's save.
|
||||
|
||||
import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker";
|
||||
import type { PerModelConfig } from "@/features/model-picker";
|
||||
import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
export function HubModelSettingsView({
|
||||
target,
|
||||
loadedConfig = null,
|
||||
loadedContextLength = null,
|
||||
onBack,
|
||||
onRun,
|
||||
compact = false,
|
||||
}: {
|
||||
target: ModelPickTarget;
|
||||
/** Non-null when this model is the loaded one, so the page can show live values. */
|
||||
loadedConfig?: PerModelConfig | null;
|
||||
loadedContextLength?: number | null;
|
||||
onBack: () => void;
|
||||
/** Apply + load with these settings. */
|
||||
onRun: (config: PerModelConfig) => void;
|
||||
compact?: boolean;
|
||||
}) {
|
||||
const scrollRef = useRef<HTMLDivElement | null>(null);
|
||||
const [scrolled, setScrolled] = useState(false);
|
||||
// Mirrors HubDetailView so this view sits at the same measure as the rest of
|
||||
// the Hub rather than introducing a third column width.
|
||||
const measure = compact
|
||||
? "mx-auto w-full max-w-[860px] px-5 sm:px-5"
|
||||
: "mx-auto w-full max-w-[1100px] px-5 sm:px-8";
|
||||
|
||||
useEffect(() => {
|
||||
const el = scrollRef.current;
|
||||
if (!el) return;
|
||||
const onScroll = () => {
|
||||
const next = el.scrollTop > 0;
|
||||
setScrolled((current) => (current === next ? current : next));
|
||||
};
|
||||
onScroll();
|
||||
el.addEventListener("scroll", onScroll, { passive: true });
|
||||
return () => el.removeEventListener("scroll", onScroll);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="relative flex min-h-0 flex-1 flex-col">
|
||||
<div
|
||||
aria-hidden="true"
|
||||
data-scrolled={scrolled || undefined}
|
||||
className="hub-scroll-fade pointer-events-none absolute inset-x-0 top-0 z-10 h-7"
|
||||
/>
|
||||
<div
|
||||
ref={scrollRef}
|
||||
data-hub-scroll="true"
|
||||
className={cn(
|
||||
"min-h-0 flex-1 overflow-x-hidden overflow-y-auto [overflow-anchor:none] [scrollbar-width:thin]",
|
||||
compact
|
||||
? "mr-2 [scrollbar-gutter:stable]"
|
||||
: "[scrollbar-gutter:stable_both-edges]",
|
||||
)}
|
||||
>
|
||||
<div
|
||||
className="hub-detail-bar sticky top-0 z-20"
|
||||
data-scrolled={scrolled || undefined}
|
||||
>
|
||||
<div className={`${measure} py-3`}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onBack}
|
||||
className="-ml-1.5 inline-flex h-8 cursor-pointer select-none items-center gap-1.5 rounded-full pl-1.5 pr-2.5 text-ui-12p5 font-medium text-muted-foreground transition-colors hover:bg-foreground/[0.05] hover:text-foreground dark:hover:bg-white/[0.06]"
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={ArrowLeft01Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-3.5"
|
||||
/>
|
||||
Back to Hub
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className={cn(measure, "pb-20")}>
|
||||
<header className="flex flex-col gap-1 pb-5">
|
||||
<h1 className="min-w-0 break-words text-ui-24 font-semibold leading-[1.1] tracking-[-0.022em] text-foreground">
|
||||
{target.displayName}
|
||||
</h1>
|
||||
<p className="min-w-0 break-all text-ui-12 text-muted-foreground">
|
||||
{target.id}
|
||||
{target.ggufVariant ? ` · ${target.ggufVariant}` : ""}
|
||||
</p>
|
||||
</header>
|
||||
|
||||
<div className="mb-5 flex items-start gap-2.5 rounded-xl border border-border/60 bg-card px-4 py-3">
|
||||
<span className="mt-0.5 flex size-7 shrink-0 items-center justify-center rounded-lg border border-border/60 bg-muted/40">
|
||||
<HugeiconsIcon
|
||||
icon={Globe02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-3.5"
|
||||
/>
|
||||
</span>
|
||||
<p className="min-w-0 text-ui-12 leading-[1.5] text-muted-foreground">
|
||||
Saved settings apply everywhere this model loads, including when an
|
||||
OpenAI-compatible API request asks for it. Turn on{" "}
|
||||
<span className="font-medium text-foreground">
|
||||
Remember for this model
|
||||
</span>{" "}
|
||||
below to keep them.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="rounded-xl border border-border/60 bg-card px-4 py-4">
|
||||
<ModelConfigPage
|
||||
key={`${target.id}::${target.ggufVariant ?? ""}`}
|
||||
target={target}
|
||||
onRun={onRun}
|
||||
loadedConfig={loadedConfig}
|
||||
loadedContextLength={loadedContextLength}
|
||||
variant="page"
|
||||
// The page heading above already names the model; the built-in
|
||||
// "Run settings" block would print it a second time.
|
||||
showHeader={false}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -53,6 +53,7 @@ import { useHfTokenStore } from "../stores/hf-token-store";
|
|||
import { DotTag } from "./dot-tag";
|
||||
import {
|
||||
CardDeleteButton,
|
||||
CardSettingsButton,
|
||||
CardUpdateButton,
|
||||
DeleteConfirmDialog,
|
||||
UpdateConfirmDialog,
|
||||
|
|
@ -96,6 +97,8 @@ interface LocalOnDeviceCardProps {
|
|||
onEject?: () => void;
|
||||
onTrain?: () => void;
|
||||
onChange?: () => void;
|
||||
/** Open this model's full settings page for the shown quant. */
|
||||
onOpenSettings?: (ggufVariant: string | null) => void;
|
||||
}
|
||||
|
||||
function formatAdapterLabel(
|
||||
|
|
@ -214,6 +217,7 @@ export function LocalOnDeviceCard({
|
|||
onEject,
|
||||
onTrain,
|
||||
onChange,
|
||||
onOpenSettings,
|
||||
}: LocalOnDeviceCardProps) {
|
||||
const [deleteOpen, setDeleteOpen] = useState(false);
|
||||
const [updateOpen, setUpdateOpen] = useState(false);
|
||||
|
|
@ -549,6 +553,14 @@ export function LocalOnDeviceCard({
|
|||
)}
|
||||
</span>
|
||||
<div className="ml-auto flex items-center gap-0.5">
|
||||
{onOpenSettings && (
|
||||
<CardSettingsButton
|
||||
label={`Settings for ${repoId}`}
|
||||
// Hand over the quant this card resolved, so the settings page
|
||||
// edits the variant the user is looking at rather than the repo.
|
||||
onClick={() => onOpenSettings(selectedQuant ?? null)}
|
||||
/>
|
||||
)}
|
||||
{canUpdate && (
|
||||
<CardUpdateButton
|
||||
label={`Update ${repoId}`}
|
||||
|
|
|
|||
|
|
@ -403,6 +403,8 @@ export type ModelInspectorActions = {
|
|||
onTrain?: () => void;
|
||||
onInventoryChange?: () => void;
|
||||
onSearchHub?: (query: string) => void;
|
||||
/** Open this model's full settings page, with the quant the card resolved. */
|
||||
onOpenSettings?: (ggufVariant: string | null) => void;
|
||||
};
|
||||
|
||||
export const ModelInspector = memo(function ModelInspector({
|
||||
|
|
@ -438,6 +440,7 @@ export const ModelInspector = memo(function ModelInspector({
|
|||
onTrain,
|
||||
onInventoryChange,
|
||||
onSearchHub,
|
||||
onOpenSettings,
|
||||
} = actions;
|
||||
const deviceType = usePlatformStore((s) => s.deviceType);
|
||||
const chatOnly = usePlatformStore((s) => s.isChatOnly());
|
||||
|
|
@ -705,6 +708,7 @@ export const ModelInspector = memo(function ModelInspector({
|
|||
model.isDownloaded && canTrainModel ? onTrain : undefined
|
||||
}
|
||||
onChange={onInventoryChange}
|
||||
onOpenSettings={onOpenSettings}
|
||||
/>
|
||||
) : (
|
||||
<DownloadSection
|
||||
|
|
|
|||
|
|
@ -263,6 +263,7 @@ export function DownloadedList({
|
|||
compact = false,
|
||||
sort,
|
||||
onInventoryChange,
|
||||
onOpenModelSettings,
|
||||
}: {
|
||||
cachedRows: CachedInventoryRow[];
|
||||
localRows: LocalInventoryRow[];
|
||||
|
|
@ -284,6 +285,8 @@ export function DownloadedList({
|
|||
compact?: boolean;
|
||||
sort: InventorySort;
|
||||
onInventoryChange?: () => void;
|
||||
/** Open a downloaded model's full settings page. */
|
||||
onOpenModelSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void;
|
||||
}) {
|
||||
// Pinned repos surface first regardless of the active sort; the chosen sort
|
||||
// still orders rows within the pinned and unpinned groups.
|
||||
|
|
@ -383,6 +386,7 @@ export function DownloadedList({
|
|||
compact={compact}
|
||||
onSelect={onSelect}
|
||||
onChange={onInventoryChange}
|
||||
onOpenSettings={onOpenModelSettings}
|
||||
/>
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -573,6 +573,7 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
compact = false,
|
||||
onSelect,
|
||||
onChange,
|
||||
onOpenSettings,
|
||||
}: {
|
||||
row: CachedInventoryRow | LocalInventoryRow;
|
||||
selected: boolean;
|
||||
|
|
@ -585,6 +586,8 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
compact?: boolean;
|
||||
onSelect: (id: string) => void;
|
||||
onChange?: () => void;
|
||||
/** Open this model's full settings page. Omitted for datasets. */
|
||||
onOpenSettings?: (row: CachedInventoryRow | LocalInventoryRow) => void;
|
||||
}) {
|
||||
const rowModelId =
|
||||
row.kind === "cache"
|
||||
|
|
@ -732,30 +735,41 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
const rowPinned =
|
||||
cacheDeletableRepoId != null &&
|
||||
pinnedKeys.includes(pinKey(cacheDeletableRepoId));
|
||||
// Settings is available for any downloaded model, not just deletable ones: a
|
||||
// model scanned from a local folder is just as configurable as a cached repo.
|
||||
// The menu therefore renders whenever either action applies, and each item
|
||||
// gates itself. `deletableRepoId` (rather than the boolean) keeps the non-null
|
||||
// narrowing the delete closures below rely on.
|
||||
const settingsAction =
|
||||
!isDataset && onOpenSettings ? { onOpen: () => onOpenSettings(row) } : undefined;
|
||||
const deletableRepoId = canDelete ? cacheDeletableRepoId : null;
|
||||
const deleteAction =
|
||||
canDelete && cacheDeletableRepoId ? (
|
||||
deletableRepoId || settingsAction ? (
|
||||
<ModelRowMenu
|
||||
ariaLabel={`More options for ${cacheDeletableRepoId}`}
|
||||
ariaLabel={`More options for ${deletableRepoId ?? rowModelId}`}
|
||||
buttonClassName="pointer-events-auto hub-modal-pe-guard p-2 opacity-0 transition-opacity group-hover/row:opacity-100 focus-visible:opacity-100 data-[state=open]:opacity-100 [@media(pointer:coarse)]:opacity-100"
|
||||
iconClassName="size-4"
|
||||
settings={settingsAction}
|
||||
pin={
|
||||
isDataset
|
||||
isDataset || !deletableRepoId
|
||||
? undefined
|
||||
: {
|
||||
pinned: rowPinned,
|
||||
pinLabel: "Pin to top",
|
||||
unpinLabel: "Unpin",
|
||||
onToggle: () => togglePinned(cacheDeletableRepoId),
|
||||
onToggle: () => togglePinned(deletableRepoId),
|
||||
}
|
||||
}
|
||||
cachePath={isDataset ? undefined : { repoId: cacheDeletableRepoId }}
|
||||
del={{
|
||||
cachePath={
|
||||
isDataset || !deletableRepoId ? undefined : { repoId: deletableRepoId }
|
||||
}
|
||||
del={deletableRepoId ? {
|
||||
title: isDataset ? "Delete cached dataset?" : "Delete cached model?",
|
||||
description: (
|
||||
<>
|
||||
This will remove{" "}
|
||||
<span className="font-medium text-foreground">
|
||||
{cacheDeletableRepoId}
|
||||
{deletableRepoId}
|
||||
</span>{" "}
|
||||
{isDataset
|
||||
? "and its downloaded files"
|
||||
|
|
@ -766,17 +780,17 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
disk. You can re-download it later.
|
||||
</>
|
||||
),
|
||||
successMessage: `Deleted ${cacheDeletableRepoId}`,
|
||||
successMessage: `Deleted ${deletableRepoId}`,
|
||||
onConfirm: async () => {
|
||||
// Delete only the copy this row shows: cache rows carry the owning
|
||||
// cache path, so pass it through and leave other caches untouched.
|
||||
const rowCachePath =
|
||||
row.kind === "cache" ? (row.cachePath ?? undefined) : undefined;
|
||||
if (isDataset) {
|
||||
await deleteCachedDataset(cacheDeletableRepoId, rowCachePath);
|
||||
await deleteCachedDataset(deletableRepoId, rowCachePath);
|
||||
} else {
|
||||
await deleteCachedModel(
|
||||
cacheDeletableRepoId,
|
||||
deletableRepoId,
|
||||
undefined,
|
||||
undefined,
|
||||
rowCachePath,
|
||||
|
|
@ -787,11 +801,11 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
usePinnedModelsStore.getState();
|
||||
for (const key of pinned) {
|
||||
if (
|
||||
key === pinKey(cacheDeletableRepoId) ||
|
||||
key.startsWith(`${cacheDeletableRepoId}::`)
|
||||
key === pinKey(deletableRepoId) ||
|
||||
key.startsWith(`${deletableRepoId}::`)
|
||||
) {
|
||||
toggle(
|
||||
cacheDeletableRepoId,
|
||||
deletableRepoId,
|
||||
key.includes("::")
|
||||
? key.slice(key.indexOf("::") + 2)
|
||||
: undefined,
|
||||
|
|
@ -801,7 +815,7 @@ export const InventoryRow = memo(function InventoryRow({
|
|||
}
|
||||
},
|
||||
onDeleted: onChange,
|
||||
}}
|
||||
} : undefined}
|
||||
/>
|
||||
) : null;
|
||||
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import {
|
|||
import type {
|
||||
CachedInventoryRow,
|
||||
DiscoverRow,
|
||||
InventoryRow,
|
||||
LocalInventoryRow,
|
||||
ModelsTab,
|
||||
} from "../types";
|
||||
|
|
@ -70,6 +71,8 @@ export interface ModelsCatalogHandlers {
|
|||
onRetry: () => void;
|
||||
onInventoryChange?: () => void;
|
||||
onSwitchDevice?: () => void;
|
||||
/** Open a downloaded model's full settings page. */
|
||||
onOpenModelSettings?: (row: InventoryRow) => void;
|
||||
}
|
||||
|
||||
function assignRef<T>(ref: RefObject<T | null>, value: T | null) {
|
||||
|
|
@ -128,6 +131,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({
|
|||
onRetry,
|
||||
onInventoryChange,
|
||||
onSwitchDevice,
|
||||
onOpenModelSettings,
|
||||
} = handlers;
|
||||
const [scrolled, setScrolled] = useState(false);
|
||||
const [streamingActive, setStreamingActive] = useState(false);
|
||||
|
|
@ -483,6 +487,7 @@ export const ModelsCatalog = memo(function ModelsCatalog({
|
|||
columns={discoverView === "two" ? 2 : 1}
|
||||
sort={inventorySort}
|
||||
onInventoryChange={onInventoryChange}
|
||||
onOpenModelSettings={onOpenModelSettings}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import {
|
|||
import {
|
||||
getInferenceStatus,
|
||||
isExternalModelId,
|
||||
listGgufVariants,
|
||||
useChatModelRuntime,
|
||||
useChatRuntimeStore,
|
||||
} from "@/features/chat";
|
||||
|
|
@ -24,9 +25,13 @@ import { ggufVariantsMatch, modelIdsMatch } from "./lib/model-identity";
|
|||
import { hfApiToken, useHfTokenStore } from "./stores/hf-token-store";
|
||||
import {
|
||||
applyModelLoadConfigToRuntime,
|
||||
applyPerModelConfigToRuntime,
|
||||
currentRuntimePerModelConfig,
|
||||
hfModelFitsDevice,
|
||||
resolveInitialConfig,
|
||||
useActiveModelConfig,
|
||||
type ModelPickTarget,
|
||||
type PerModelConfig,
|
||||
} from "@/features/model-picker";
|
||||
import { useDebouncedValue } from "@/hooks/use-debounced-value";
|
||||
import { useGpuInfo } from "@/hooks/use-gpu-info";
|
||||
|
|
@ -42,6 +47,7 @@ import {
|
|||
} from "react";
|
||||
import { ExternalLinkConfirmDialog } from "./catalog/external-link-confirm-dialog";
|
||||
import { HubDetailView } from "./catalog/hub-detail-view";
|
||||
import { HubModelSettingsView } from "./catalog/hub-model-settings-view";
|
||||
import { HubFeed } from "./catalog/hub-feed";
|
||||
import { HubTopBar } from "./catalog/hub-top-bar";
|
||||
import {
|
||||
|
|
@ -345,6 +351,12 @@ export function ModelsPage() {
|
|||
const activeCheckpoint =
|
||||
checkpoint && !isExternalModelId(checkpoint) ? checkpoint : null;
|
||||
const activeGgufVariant = useChatRuntimeStore((s) => s.activeGgufVariant);
|
||||
const activeGgufContextLength = useChatRuntimeStore(
|
||||
(s) => s.ggufContextLength,
|
||||
);
|
||||
// Live settings of the loaded model, so opening its settings page shows what
|
||||
// it is actually running with rather than the last saved draft.
|
||||
const { config: activeModelConfig } = useActiveModelConfig();
|
||||
// Shared with the chat model selector: list only models sized for this device.
|
||||
const fitOnDeviceOnly = useChatRuntimeStore((s) => s.fitOnDeviceOnly);
|
||||
const setFitOnDeviceOnly = useChatRuntimeStore((s) => s.setFitOnDeviceOnly);
|
||||
|
|
@ -1213,6 +1225,100 @@ export function ModelsPage() {
|
|||
runSelectedModel(opts, selectedModel?.isDownloaded ?? true),
|
||||
[runSelectedModel, selectedModel],
|
||||
);
|
||||
|
||||
// Full-page per-model settings, opened from a downloaded row's menu. Local
|
||||
// state rather than a URL param: the page is a transient editor over the
|
||||
// catalog, and a deep link to it would need the row's identity re-resolved
|
||||
// against an inventory that may not have loaded yet.
|
||||
const [settingsTarget, setSettingsTarget] = useState<ModelPickTarget | null>(
|
||||
null,
|
||||
);
|
||||
const openModelSettings = useCallback(
|
||||
async (row: CachedInventoryRow | LocalInventoryRow) => {
|
||||
// loadId is what the loader accepts; repoId is only a display/API alias.
|
||||
const id = row.loadId;
|
||||
// Cached repo rows never carry a quant: the inventory emits one row per
|
||||
// repo with format_variant null (see cache_inventory.py). Opening settings
|
||||
// with a null variant would key the saved config to `repo::` while the
|
||||
// loader reads `repo::Q4_K_M`, so the settings would silently never apply
|
||||
// and the server mirror would be keyed wrong too. Resolve the quant the
|
||||
// same way the on-device card does before opening.
|
||||
let ggufVariant = row.formatVariant?.trim() || null;
|
||||
if (!ggufVariant && row.isGguf && row.capabilities.requiresVariant) {
|
||||
const repoId = row.kind === "cache" ? row.repoId : (row.repoId ?? null);
|
||||
if (repoId) {
|
||||
try {
|
||||
const res = await listGgufVariants(repoId, hfApiToken(hfToken), {
|
||||
preferLocalCache: true,
|
||||
localPath: row.kind === "local" ? row.path : (row.cachePath ?? null),
|
||||
});
|
||||
const downloaded = res.variants.filter((v) => v.downloaded);
|
||||
ggufVariant =
|
||||
// Prefer the loaded quant, then the repo default, then whatever is
|
||||
// on disk, mirroring LocalOnDeviceCard's selectedQuant.
|
||||
downloaded.find((v) =>
|
||||
ggufVariantsMatch(v.quant, activeGgufVariant),
|
||||
)?.quant ??
|
||||
downloaded.find((v) =>
|
||||
ggufVariantsMatch(v.quant, res.default_variant),
|
||||
)?.quant ??
|
||||
downloaded[0]?.quant ??
|
||||
null;
|
||||
} catch {
|
||||
// Offline or an unreadable cache: fall through with no variant. The
|
||||
// settings page still works, it just cannot pin a specific quant.
|
||||
ggufVariant = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id;
|
||||
setSettingsTarget({
|
||||
id,
|
||||
displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf,
|
||||
ggufVariant,
|
||||
isGguf: row.isGguf,
|
||||
meta: {
|
||||
source: "local",
|
||||
isLora: row.modelFormat === "adapter",
|
||||
ggufVariant: ggufVariant ?? undefined,
|
||||
isGguf: row.isGguf,
|
||||
// Partial downloads still open settings, but must not claim to be
|
||||
// complete or the loader skips its download-progress reporting.
|
||||
isDownloaded: !row.partial,
|
||||
// Not carried on inventory rows; ModelConfigPage reads the GGUF header
|
||||
// itself to size the context slider.
|
||||
contextLength: null,
|
||||
},
|
||||
});
|
||||
},
|
||||
[activeGgufVariant, hfToken],
|
||||
);
|
||||
// Applying from the settings page loads the model with exactly those settings.
|
||||
// ModelConfigPage has already persisted them (locally and, when "remember" is
|
||||
// on, to the server), so an API request for this model gets the same load.
|
||||
const runSettingsTarget = useCallback(
|
||||
(config: PerModelConfig) => {
|
||||
const target = settingsTarget;
|
||||
if (!target) return;
|
||||
const previousConfig = currentRuntimePerModelConfig({
|
||||
includeMaxSeqLength: true,
|
||||
});
|
||||
applyPerModelConfigToRuntime(config);
|
||||
setSettingsTarget(null);
|
||||
void selectModel({
|
||||
id: target.id,
|
||||
source: "local",
|
||||
ggufVariant: target.ggufVariant ?? undefined,
|
||||
isGguf: target.isGguf,
|
||||
isDownloaded: true,
|
||||
isLora: target.meta.isLora,
|
||||
keepSpeculative: true,
|
||||
forceReload: true,
|
||||
previousConfig,
|
||||
}).catch(() => undefined);
|
||||
},
|
||||
[selectModel, settingsTarget],
|
||||
);
|
||||
const handleLoadLocal = useCallback(
|
||||
(opts: ModelLoadOptions = {}) => runSelectedModel(opts, true),
|
||||
[runSelectedModel],
|
||||
|
|
@ -1220,6 +1326,30 @@ export function ModelsPage() {
|
|||
const handleTrain = useCallback(() => {
|
||||
// Hub → train integration ships in a later PR.
|
||||
}, []);
|
||||
// Settings opened from the detail view's on-device card. The card resolves
|
||||
// which quant it is showing, so it passes that in rather than re-deriving it.
|
||||
const openSelectedModelSettings = useCallback(
|
||||
(ggufVariant: string | null) => {
|
||||
if (!selectedModel) return;
|
||||
const id = selectedModel.resource.runId;
|
||||
const leaf = id.split(/[\\/]/).filter(Boolean).pop() ?? id;
|
||||
setSettingsTarget({
|
||||
id,
|
||||
displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf,
|
||||
ggufVariant,
|
||||
isGguf: selectedModel.isGguf,
|
||||
meta: {
|
||||
source: "local",
|
||||
isLora: selectedModel.modelFormat === "adapter",
|
||||
ggufVariant: ggufVariant ?? undefined,
|
||||
isGguf: selectedModel.isGguf,
|
||||
isDownloaded: selectedModel.isDownloaded,
|
||||
contextLength: null,
|
||||
},
|
||||
});
|
||||
},
|
||||
[selectedModel],
|
||||
);
|
||||
const handleSearchHub = useCallback(
|
||||
(next: string) => {
|
||||
const trimmed = next.trim();
|
||||
|
|
@ -1280,6 +1410,7 @@ export function ModelsPage() {
|
|||
onTrain: handleTrain,
|
||||
onInventoryChange: refreshInventory,
|
||||
onSearchHub: handleSearchHub,
|
||||
onOpenSettings: openSelectedModelSettings,
|
||||
}),
|
||||
[
|
||||
handleLoad,
|
||||
|
|
@ -1289,6 +1420,7 @@ export function ModelsPage() {
|
|||
handleTrain,
|
||||
handleSearchHub,
|
||||
refreshInventory,
|
||||
openSelectedModelSettings,
|
||||
],
|
||||
);
|
||||
|
||||
|
|
@ -1370,6 +1502,7 @@ export function ModelsPage() {
|
|||
onRetry: handleRetrySearch,
|
||||
onInventoryChange: refreshInventory,
|
||||
onSwitchDevice: handleSwitchDevice,
|
||||
onOpenModelSettings: openModelSettings,
|
||||
}),
|
||||
[
|
||||
handleSelect,
|
||||
|
|
@ -1378,6 +1511,7 @@ export function ModelsPage() {
|
|||
handleRetrySearch,
|
||||
refreshInventory,
|
||||
handleSwitchDevice,
|
||||
openModelSettings,
|
||||
],
|
||||
);
|
||||
|
||||
|
|
@ -1520,6 +1654,10 @@ export function ModelsPage() {
|
|||
|
||||
const detailOpen = urlModel !== null;
|
||||
const splitMode = allModelsView === "split";
|
||||
// The catalog is unreachable when an opaque overlay sits on top of it: the
|
||||
// detail view (full-page layout only, since split renders it alongside) or the
|
||||
// settings page (always full-bleed).
|
||||
const catalogCovered = (detailOpen && !splitMode) || settingsTarget !== null;
|
||||
|
||||
return (
|
||||
<div className="hub-page flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-hidden bg-background">
|
||||
|
|
@ -1575,9 +1713,15 @@ export function ModelsPage() {
|
|||
splitMode
|
||||
? "flex-1 lg:w-[460px] lg:max-w-[44%] lg:flex-none lg:shrink-0 lg:border-r lg:border-border/60"
|
||||
: "flex-1",
|
||||
detailOpen && !splitMode && "pointer-events-none",
|
||||
// The settings page is a full-bleed opaque overlay in every layout,
|
||||
// including split, so it always takes the catalog out of the tab
|
||||
// order. Without this, tabbing out of the settings form walks into
|
||||
// the virtualized rows hidden behind it and screen readers announce
|
||||
// the whole model list underneath.
|
||||
catalogCovered && "pointer-events-none",
|
||||
)}
|
||||
aria-hidden={(detailOpen && !splitMode) || undefined}
|
||||
aria-hidden={catalogCovered || undefined}
|
||||
inert={catalogCovered || undefined}
|
||||
>
|
||||
<ModelsCatalog
|
||||
state={catalogState}
|
||||
|
|
@ -1625,6 +1769,32 @@ export function ModelsPage() {
|
|||
</div>
|
||||
)
|
||||
)}
|
||||
|
||||
{/* Sits above the detail overlay (z-30): opening settings from a row
|
||||
while a model preview is open should show the settings, not stack
|
||||
behind it. */}
|
||||
{settingsTarget && (
|
||||
<div className="hub-canvas absolute inset-0 z-30 flex min-h-0 flex-col">
|
||||
<HubModelSettingsView
|
||||
target={settingsTarget}
|
||||
loadedConfig={
|
||||
modelIdsMatch(activeCheckpoint, settingsTarget.id) &&
|
||||
ggufVariantsMatch(activeGgufVariant, settingsTarget.ggufVariant)
|
||||
? activeModelConfig
|
||||
: null
|
||||
}
|
||||
loadedContextLength={
|
||||
modelIdsMatch(activeCheckpoint, settingsTarget.id) &&
|
||||
ggufVariantsMatch(activeGgufVariant, settingsTarget.ggufVariant)
|
||||
? activeGgufContextLength
|
||||
: null
|
||||
}
|
||||
onBack={() => setSettingsTarget(null)}
|
||||
onRun={runSettingsTarget}
|
||||
compact={splitMode}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<OnDeviceFoldersDialog
|
||||
|
|
|
|||
174
studio/frontend/src/features/model-picker/api/model-overrides.ts
Normal file
174
studio/frontend/src/features/model-picker/api/model-overrides.ts
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// Server-side mirror of the per-model config.
|
||||
//
|
||||
// The per-model config in ../model-config/per-model-config.ts lives in browser
|
||||
// localStorage, so it only ever applied to loads the browser made. A model loaded
|
||||
// by an OpenAI-compatible API request (Model auto-switch) is loaded by the backend
|
||||
// with no browser in the loop, which is why an API load used to come up with none
|
||||
// of the settings the user had configured. Mirroring every save to the backend's
|
||||
// override map closes that gap: routes/inference.py reads it on the auto-switch
|
||||
// path and rebuilds the same LoadRequest the picker would have sent.
|
||||
|
||||
import { authFetch } from "@/features/auth";
|
||||
import { readFastApiError } from "@/lib/format-fastapi-error";
|
||||
import type { PerModelConfig } from "../model-config/per-model-config";
|
||||
|
||||
const OVERRIDES_URL = "/api/settings/openai-auto-switch/overrides";
|
||||
|
||||
/** One model's stored launch config, as the backend persists it. */
|
||||
export interface ApiModelOverride {
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
llama_extra_args?: string[];
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
max_seq_length?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
custom_context_length?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
kv_cache_dtype?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
speculative_type?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
spec_draft_n_max?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
tensor_parallel?: boolean;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
chat_template_override?: string;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
gpu_memory_mode?: "auto" | "manual";
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
gpu_layers?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
n_cpu_moe?: number;
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
gpu_ids?: number[];
|
||||
}
|
||||
|
||||
export type ApiModelOverrides = Record<string, ApiModelOverride>;
|
||||
|
||||
/**
|
||||
* The key one model's config is stored under.
|
||||
*
|
||||
* Uses the `repo:VARIANT` form an OpenAI request names a quant by, so two quants
|
||||
* of the same repo keep separate configs and the backend can match the requested
|
||||
* model name directly. Falls back to the bare id when there is no variant.
|
||||
*/
|
||||
export function modelOverrideKey(
|
||||
modelId: string,
|
||||
ggufVariant?: string | null,
|
||||
): string {
|
||||
return ggufVariant ? `${modelId}:${ggufVariant}` : modelId;
|
||||
}
|
||||
|
||||
export async function fetchModelOverrides(): Promise<ApiModelOverrides> {
|
||||
const res = await authFetch(OVERRIDES_URL);
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
await readFastApiError(res, "Failed to load saved model settings"),
|
||||
);
|
||||
}
|
||||
const body = (await res.json()) as { overrides?: ApiModelOverrides };
|
||||
return body.overrides ?? {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Translate the UI's per-model config into the backend's schema.
|
||||
*
|
||||
* Only fields the user actually set are sent: the backend reads an absent field
|
||||
* as "use the app default", so sending nulls would pin defaults and stop the
|
||||
* model following later changes to the global preferences. `null` config means
|
||||
* "no saved settings", which clears the entry.
|
||||
*/
|
||||
function toApiOverride(config: PerModelConfig | null): ApiModelOverride {
|
||||
if (!config) {
|
||||
return {};
|
||||
}
|
||||
const payload: ApiModelOverride = {};
|
||||
if (config.maxSeqLength && config.maxSeqLength > 0) {
|
||||
payload.max_seq_length = config.maxSeqLength;
|
||||
}
|
||||
if (config.customContextLength && config.customContextLength > 0) {
|
||||
payload.custom_context_length = config.customContextLength;
|
||||
}
|
||||
if (config.kvCacheDtype) {
|
||||
payload.kv_cache_dtype = config.kvCacheDtype;
|
||||
}
|
||||
if (config.speculativeType) {
|
||||
payload.speculative_type = config.speculativeType;
|
||||
}
|
||||
if (config.specDraftNMax && config.specDraftNMax > 0) {
|
||||
payload.spec_draft_n_max = config.specDraftNMax;
|
||||
}
|
||||
if (config.tensorParallel) {
|
||||
payload.tensor_parallel = true;
|
||||
}
|
||||
if (config.chatTemplateOverride?.trim()) {
|
||||
payload.chat_template_override = config.chatTemplateOverride;
|
||||
}
|
||||
// Only "manual" is a real override; "auto" is the follow-the-global default.
|
||||
if (config.gpuMemoryMode === "manual") {
|
||||
payload.gpu_memory_mode = "manual";
|
||||
}
|
||||
// gpuLayers < 0 is Auto, which is also the default.
|
||||
if (typeof config.gpuLayers === "number" && config.gpuLayers >= 0) {
|
||||
payload.gpu_layers = config.gpuLayers;
|
||||
}
|
||||
if (typeof config.nCpuMoe === "number" && config.nCpuMoe > 0) {
|
||||
payload.n_cpu_moe = config.nCpuMoe;
|
||||
}
|
||||
if (config.selectedGpuIds && config.selectedGpuIds.length > 0) {
|
||||
payload.gpu_ids = config.selectedGpuIds;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
export async function putModelOverride(
|
||||
modelId: string,
|
||||
ggufVariant: string | null | undefined,
|
||||
config: PerModelConfig | null,
|
||||
): Promise<void> {
|
||||
const res = await authFetch(OVERRIDES_URL, {
|
||||
method: "PUT",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
model_id: modelOverrideKey(modelId, ggufVariant),
|
||||
// Launch flags have no UI control, so the backend preserves them when the
|
||||
// field is omitted. Forgetting a model means forgetting all of it, so that
|
||||
// path sends an explicit empty list to clear them.
|
||||
// biome-ignore lint/style/useNamingConvention: API schema
|
||||
...(config === null ? { llama_extra_args: [] } : {}),
|
||||
...toApiOverride(config),
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
await readFastApiError(res, "Failed to save model settings for the API"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mirror a per-model config save to the backend without blocking the UI.
|
||||
*
|
||||
* Deliberately best-effort: the localStorage write is the source of truth for
|
||||
* this browser and has already happened by the time this runs, so a failed sync
|
||||
* must not fail the save or interrupt a model load. It is logged rather than
|
||||
* toasted -- the only consequence is that an API-triggered load of this model
|
||||
* falls back to app defaults until the next successful save.
|
||||
*/
|
||||
export function syncModelOverride(
|
||||
modelId: string,
|
||||
ggufVariant: string | null | undefined,
|
||||
config: PerModelConfig | null,
|
||||
): void {
|
||||
void putModelOverride(modelId, ggufVariant, config).catch(
|
||||
(error: unknown) => {
|
||||
console.warn(
|
||||
"Failed to mirror model settings to the server; an API load of this model will use defaults.",
|
||||
error,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -55,6 +55,7 @@ import {
|
|||
resolveInitialConfig,
|
||||
savePerModelConfig,
|
||||
} from "../model-config/per-model-config";
|
||||
import { syncModelOverride } from "../api/model-overrides";
|
||||
import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog";
|
||||
import type { ModelPickTarget } from "./model-selector/types";
|
||||
import {
|
||||
|
|
@ -578,6 +579,12 @@ interface ModelConfigPageProps {
|
|||
loadedContextLength?: number | null;
|
||||
initialConfig?: PerModelConfig | null;
|
||||
variant?: "page" | "sidebar";
|
||||
/**
|
||||
* Page variant only: render the built-in "Run settings" title block. A host
|
||||
* that already shows the model name as its own page heading (the Hub's
|
||||
* settings page) turns this off so the name is not printed twice.
|
||||
*/
|
||||
showHeader?: boolean;
|
||||
}
|
||||
|
||||
export function ModelConfigPage({
|
||||
|
|
@ -588,6 +595,7 @@ export function ModelConfigPage({
|
|||
loadedContextLength = null,
|
||||
initialConfig = null,
|
||||
variant = "page",
|
||||
showHeader = true,
|
||||
}: ModelConfigPageProps) {
|
||||
const rememberId = useId();
|
||||
const isActiveModel = loadedConfig != null;
|
||||
|
|
@ -870,6 +878,21 @@ export function ModelConfigPage({
|
|||
} else {
|
||||
saveFailed = !deletePerModelConfig(target.id, target.ggufVariant);
|
||||
}
|
||||
// Mirror to the server so an OpenAI-compatible API request that loads this
|
||||
// model gets these exact settings, not app defaults. Best-effort and
|
||||
// non-blocking: the localStorage write above already governs this browser.
|
||||
// Forgetting clears the server entry too, so the two never disagree.
|
||||
//
|
||||
// Skipped when the local write failed (quota, a future-schema entry): the
|
||||
// browser and the server would otherwise permanently disagree about this
|
||||
// model, with no way for the user to tell which one the next load used.
|
||||
if (!saveFailed) {
|
||||
syncModelOverride(
|
||||
target.id,
|
||||
target.ggufVariant,
|
||||
remember ? effectiveRuntimeConfig : null,
|
||||
);
|
||||
}
|
||||
if (effectivePersistenceOnly) {
|
||||
if (saveFailed) {
|
||||
toast.error("Couldn't save settings for this model.");
|
||||
|
|
@ -898,7 +921,7 @@ export function ModelConfigPage({
|
|||
|
||||
return (
|
||||
<div className="flex flex-col">
|
||||
{variant === "page" && (
|
||||
{variant === "page" && showHeader && (
|
||||
<div className="flex items-center gap-2.5 pb-4">
|
||||
{onBack && (
|
||||
<button
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ import {
|
|||
MoreVerticalIcon,
|
||||
PinIcon,
|
||||
PinOffIcon,
|
||||
Settings02Icon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { RefreshCw } from "lucide-react";
|
||||
|
|
@ -73,11 +74,17 @@ interface ModelRowMenuCachePath {
|
|||
variant?: string;
|
||||
}
|
||||
|
||||
/** Opens the model's own settings page (load config + what the API will apply). */
|
||||
interface ModelRowMenuSettings {
|
||||
onOpen: () => void;
|
||||
}
|
||||
|
||||
export function ModelRowMenu({
|
||||
ariaLabel,
|
||||
buttonClassName,
|
||||
iconClassName,
|
||||
cachePath,
|
||||
settings,
|
||||
pin,
|
||||
update,
|
||||
del,
|
||||
|
|
@ -87,6 +94,8 @@ export function ModelRowMenu({
|
|||
iconClassName?: string;
|
||||
/** Enables "Reveal in Finder" for cached repos. */
|
||||
cachePath?: ModelRowMenuCachePath;
|
||||
/** Opens this model's full settings page. */
|
||||
settings?: ModelRowMenuSettings;
|
||||
pin?: ModelRowMenuPin;
|
||||
update?: ModelRowMenuUpdate;
|
||||
del?: ModelRowMenuDelete;
|
||||
|
|
@ -167,7 +176,7 @@ export function ModelRowMenu({
|
|||
});
|
||||
}, [cachePathRepoId, cachePathVariant]);
|
||||
|
||||
if (!pin && !update && !del && !cachePath) return null;
|
||||
if (!pin && !update && !del && !cachePath && !settings) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
|
|
@ -195,6 +204,21 @@ export function ModelRowMenu({
|
|||
sideOffset={2}
|
||||
className="unsloth-plus-menu menu-flat-destructive w-48"
|
||||
>
|
||||
{settings && (
|
||||
<DropdownMenuItem
|
||||
onSelect={(e) => {
|
||||
e.stopPropagation();
|
||||
settings.onOpen();
|
||||
}}
|
||||
>
|
||||
<HugeiconsIcon
|
||||
icon={Settings02Icon}
|
||||
strokeWidth={1.75}
|
||||
className="size-icon"
|
||||
/>
|
||||
<span>Settings</span>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{pin && (
|
||||
<DropdownMenuItem
|
||||
onSelect={(e) => {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,17 @@ export {
|
|||
type NumericValueInputHandle,
|
||||
snapToStep,
|
||||
} from "./components/numeric-value-input";
|
||||
export { ModelConfigPage } from "./components/model-config-page";
|
||||
export { SidebarModelConfig } from "./components/sidebar-model-config";
|
||||
export type { ModelPickTarget } from "./components/model-selector/types";
|
||||
export {
|
||||
fetchModelOverrides,
|
||||
modelOverrideKey,
|
||||
putModelOverride,
|
||||
syncModelOverride,
|
||||
type ApiModelOverride,
|
||||
type ApiModelOverrides,
|
||||
} from "./api/model-overrides";
|
||||
export {
|
||||
useActiveModelConfig,
|
||||
} from "./hooks/use-active-model-config";
|
||||
|
|
|
|||
|
|
@ -1,393 +0,0 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { cn } from "@/lib/utils";
|
||||
import {
|
||||
ActivityIcon,
|
||||
ChevronDownIcon,
|
||||
CircleIcon,
|
||||
RefreshCwIcon,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
type ReactElement,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { getApiMonitor, getApiMonitorEntry } from "../../chat/api/chat-api";
|
||||
import type { ApiMonitorEntry, ApiMonitorResponse } from "../../chat/types/api";
|
||||
|
||||
const API_INFERENCE_PREFIX_RE = /^\/api\/inference/;
|
||||
const V1_PREFIX_RE = /^\/v1\//;
|
||||
|
||||
function formatTime(value: number): string {
|
||||
return new Date(value * 1000).toLocaleTimeString([], {
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
}
|
||||
|
||||
function formatDuration(value?: number | null): string {
|
||||
if (value == null) {
|
||||
return "Running";
|
||||
}
|
||||
if (value < 1000) {
|
||||
return `${value} ms`;
|
||||
}
|
||||
return `${(value / 1000).toFixed(value < 10000 ? 1 : 0)} s`;
|
||||
}
|
||||
|
||||
function formatTokens(entry: ApiMonitorEntry): string {
|
||||
if (entry.total_tokens != null) {
|
||||
return `${entry.total_tokens.toLocaleString()} tokens`;
|
||||
}
|
||||
if (entry.prompt_tokens != null || entry.completion_tokens != null) {
|
||||
const prompt = entry.prompt_tokens ?? 0;
|
||||
const completion = entry.completion_tokens ?? 0;
|
||||
return `${(prompt + completion).toLocaleString()} tokens`;
|
||||
}
|
||||
return "Tokens pending";
|
||||
}
|
||||
|
||||
function compactEndpoint(endpoint: string): string {
|
||||
return endpoint
|
||||
.replace(API_INFERENCE_PREFIX_RE, "/api")
|
||||
.replace(V1_PREFIX_RE, "/");
|
||||
}
|
||||
|
||||
function statusTone(status: ApiMonitorEntry["status"]): string {
|
||||
if (status === "running") {
|
||||
return "text-emerald-500";
|
||||
}
|
||||
if (status === "error") {
|
||||
return "text-destructive";
|
||||
}
|
||||
if (status === "cancelled") {
|
||||
return "text-amber-500";
|
||||
}
|
||||
return "text-muted-foreground";
|
||||
}
|
||||
|
||||
function UsageBar({ value }: { value?: number | null }): ReactElement | null {
|
||||
if (value == null) {
|
||||
return null;
|
||||
}
|
||||
const pct = Math.max(0, Math.min(100, Math.round(value * 100)));
|
||||
return (
|
||||
<div className="mt-2 h-1.5 w-full overflow-hidden rounded-full bg-muted">
|
||||
<div
|
||||
className="h-full rounded-full bg-control-accent"
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MonitorEntry({
|
||||
entry,
|
||||
detail,
|
||||
expanded,
|
||||
loading,
|
||||
onToggle,
|
||||
}: {
|
||||
entry: ApiMonitorEntry;
|
||||
detail?: ApiMonitorEntry;
|
||||
expanded: boolean;
|
||||
loading: boolean;
|
||||
onToggle: () => void;
|
||||
}): ReactElement {
|
||||
const hasCurrentDetail =
|
||||
detail &&
|
||||
detail.status === entry.status &&
|
||||
detail.updated_at >= entry.updated_at;
|
||||
const prompt = detail?.prompt ?? entry.prompt_preview;
|
||||
const replyText = hasCurrentDetail
|
||||
? detail.error ?? detail.reply ?? entry.error ?? entry.reply_preview
|
||||
: entry.error ?? entry.reply_preview;
|
||||
const reply = replyText || (entry.status === "running" ? "Waiting..." : "No reply");
|
||||
|
||||
return (
|
||||
<article className="min-w-0 rounded-lg border border-border/70 bg-background">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggle}
|
||||
className="flex w-full min-w-0 items-start justify-between gap-3 p-3 text-left"
|
||||
aria-expanded={expanded}
|
||||
>
|
||||
<div className="min-w-0">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<CircleIcon
|
||||
className={cn("size-2.5 fill-current", statusTone(entry.status))}
|
||||
/>
|
||||
<span className="truncate text-xs font-medium">
|
||||
{compactEndpoint(entry.endpoint)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1 truncate text-ui-11 text-muted-foreground">
|
||||
{entry.model}
|
||||
</div>
|
||||
<div className="mt-2 line-clamp-2 whitespace-pre-wrap break-words text-xs text-muted-foreground">
|
||||
{entry.error ||
|
||||
entry.reply_preview ||
|
||||
entry.prompt_preview ||
|
||||
(entry.status === "running" ? "Waiting..." : "No preview")}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-start gap-2 text-right text-ui-11 text-muted-foreground">
|
||||
<div>
|
||||
<div>{formatTime(entry.started_at)}</div>
|
||||
<div>{formatDuration(entry.duration_ms)}</div>
|
||||
</div>
|
||||
<ChevronDownIcon
|
||||
className={cn(
|
||||
"mt-0.5 size-3.5 transition-transform",
|
||||
expanded && "rotate-180",
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</button>
|
||||
|
||||
{expanded ? (
|
||||
<div className="border-t border-border/60 p-3 pt-2">
|
||||
<div className="grid gap-2">
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between gap-2 text-ui-10 font-semibold uppercase text-muted-foreground">
|
||||
<span>Prompt</span>
|
||||
{entry.prompt_truncated && !detail ? <span>Preview</span> : null}
|
||||
</div>
|
||||
<pre className="max-h-44 overflow-auto whitespace-pre-wrap break-words rounded-md bg-muted/45 p-2 text-xs leading-5">
|
||||
{loading && !detail ? "Loading..." : prompt || "No prompt text"}
|
||||
</pre>
|
||||
</div>
|
||||
<div>
|
||||
<div className="mb-1 flex items-center justify-between gap-2 text-ui-10 font-semibold uppercase text-muted-foreground">
|
||||
<span>Reply</span>
|
||||
{entry.reply_truncated && !detail ? <span>Preview</span> : null}
|
||||
</div>
|
||||
<pre className="max-h-44 overflow-auto whitespace-pre-wrap break-words rounded-md bg-muted/45 p-2 text-xs leading-5">
|
||||
{loading && !detail ? "Loading..." : reply}
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="mt-3 text-ui-11 text-muted-foreground">
|
||||
{formatTokens(entry)}
|
||||
{entry.context_length ? (
|
||||
<> / {entry.context_length.toLocaleString()} context</>
|
||||
) : null}
|
||||
<UsageBar value={entry.context_usage} />
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
</article>
|
||||
);
|
||||
}
|
||||
|
||||
export function ApiMonitorConsole(): ReactElement {
|
||||
const [data, setData] = useState<ApiMonitorResponse | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [expandedIds, setExpandedIds] = useState<Set<string>>(() => new Set());
|
||||
const [details, setDetails] = useState<Record<string, ApiMonitorEntry>>({});
|
||||
const [loadingDetails, setLoadingDetails] = useState<Set<string>>(
|
||||
() => new Set(),
|
||||
);
|
||||
const loadingDetailsRef = useRef<Set<string>>(new Set());
|
||||
const detailsRef = useRef<Record<string, ApiMonitorEntry>>({});
|
||||
|
||||
const loadMonitor = useCallback(async (): Promise<void> => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
setData(await getApiMonitor());
|
||||
setError(null);
|
||||
} catch (err: unknown) {
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
let timer: number | undefined;
|
||||
|
||||
function schedule(): void {
|
||||
timer = window.setTimeout(poll, 1500);
|
||||
}
|
||||
|
||||
function poll(): void {
|
||||
getApiMonitor()
|
||||
.then((next) => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
setData(next);
|
||||
setError(null);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
if (cancelled) {
|
||||
return;
|
||||
}
|
||||
setError(err instanceof Error ? err.message : "Monitor unavailable");
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) {
|
||||
schedule();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
poll();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
if (timer !== undefined) {
|
||||
window.clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
}, []);
|
||||
|
||||
const statusLabel = data?.status ?? "idle";
|
||||
const hasActive = (data?.active_requests ?? 0) > 0;
|
||||
const entries = useMemo(() => data?.entries ?? [], [data]);
|
||||
const loadDetail = useCallback(
|
||||
(id: string): void => {
|
||||
if (loadingDetailsRef.current.has(id)) {
|
||||
return;
|
||||
}
|
||||
loadingDetailsRef.current.add(id);
|
||||
setLoadingDetails((prev) => new Set(prev).add(id));
|
||||
getApiMonitorEntry(id)
|
||||
.then((entry) => {
|
||||
setDetails((prev) => {
|
||||
const next = { ...prev, [id]: entry };
|
||||
detailsRef.current = next;
|
||||
return next;
|
||||
});
|
||||
})
|
||||
.catch(() => {
|
||||
setDetails((prev) => {
|
||||
const next = { ...prev };
|
||||
delete next[id];
|
||||
detailsRef.current = next;
|
||||
return next;
|
||||
});
|
||||
})
|
||||
.finally(() => {
|
||||
loadingDetailsRef.current.delete(id);
|
||||
setLoadingDetails((prev) => {
|
||||
const next = new Set(prev);
|
||||
next.delete(id);
|
||||
return next;
|
||||
});
|
||||
});
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
const toggleEntry = useCallback(
|
||||
(entry: ApiMonitorEntry): void => {
|
||||
setExpandedIds((prev) => {
|
||||
const next = new Set(prev);
|
||||
if (next.has(entry.id)) {
|
||||
next.delete(entry.id);
|
||||
} else {
|
||||
next.add(entry.id);
|
||||
loadDetail(entry.id);
|
||||
}
|
||||
return next;
|
||||
});
|
||||
},
|
||||
[loadDetail],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
for (const entry of entries) {
|
||||
if (!expandedIds.has(entry.id)) {
|
||||
continue;
|
||||
}
|
||||
const cached = detailsRef.current[entry.id];
|
||||
if (!cached || cached.status !== entry.status || entry.status === "running") {
|
||||
loadDetail(entry.id);
|
||||
}
|
||||
}
|
||||
}, [entries, expandedIds, loadDetail]);
|
||||
|
||||
return (
|
||||
<section className="flex min-w-0 flex-col rounded-lg border border-border/70 bg-background">
|
||||
<div className="flex min-w-0 items-start justify-between gap-3 border-b border-border/60 px-4 py-3">
|
||||
<div className="flex min-w-0 gap-3">
|
||||
<div className="relative mt-0.5 flex size-8 shrink-0 items-center justify-center rounded-md border border-border/70 bg-muted/40">
|
||||
<ActivityIcon className="size-4 text-foreground" />
|
||||
{hasActive ? (
|
||||
<span className="absolute right-1 top-1 size-2 rounded-full bg-emerald-500" />
|
||||
) : null}
|
||||
</div>
|
||||
<div className="min-w-0">
|
||||
<h2 className="text-sm font-semibold text-foreground">
|
||||
API monitor
|
||||
</h2>
|
||||
<p className="truncate text-xs text-muted-foreground">
|
||||
{data?.active_model ?? "No model loaded"}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2">
|
||||
<div className="rounded-full border border-border px-2.5 py-1 text-xs capitalize text-muted-foreground">
|
||||
{statusLabel}
|
||||
</div>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => void loadMonitor()}
|
||||
disabled={refreshing}
|
||||
>
|
||||
<RefreshCwIcon
|
||||
className={cn("size-3.5", refreshing && "animate-spin")}
|
||||
/>
|
||||
Refresh
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between border-b border-border/60 px-4 py-2 text-xs text-muted-foreground">
|
||||
<span>
|
||||
{(data?.active_requests ?? 0).toLocaleString()} active /{" "}
|
||||
{entries.length.toLocaleString()} recent
|
||||
</span>
|
||||
{data?.context_length ? (
|
||||
<span>{data.context_length.toLocaleString()} context</span>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
<div className="max-h-[420px] min-h-24 overflow-y-auto p-3">
|
||||
{error ? (
|
||||
<div className="rounded-lg border border-destructive/40 bg-destructive/10 p-3 text-sm text-destructive">
|
||||
{error}
|
||||
</div>
|
||||
) : entries.length === 0 ? (
|
||||
<div className="rounded-lg border border-border/70 p-4 text-sm text-muted-foreground">
|
||||
No API traffic yet
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid gap-3">
|
||||
{entries.map((entry) => (
|
||||
<MonitorEntry
|
||||
key={entry.id}
|
||||
entry={entry}
|
||||
detail={details[entry.id]}
|
||||
expanded={expandedIds.has(entry.id)}
|
||||
loading={loadingDetails.has(entry.id)}
|
||||
onToggle={() => toggleEntry(entry)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
// The monitor moved out of this tab and onto its own page. It is normally
|
||||
// reached from the floating panel; this card is the way in from Settings.
|
||||
|
||||
import { Switch } from "@/components/ui/switch";
|
||||
import { useApiMonitorOverlayStore } from "@/features/api-monitor";
|
||||
import { getApiMonitor } from "@/features/chat/api/chat-api";
|
||||
import type { ApiMonitorResponse } from "@/features/chat/types/api";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ActivityIcon, ArrowRight02Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { useNavigate } from "@tanstack/react-router";
|
||||
import { type ReactElement, useEffect, useState } from "react";
|
||||
import { useSettingsDialogStore } from "../stores/settings-dialog-store";
|
||||
|
||||
export function MonitorLink(): ReactElement {
|
||||
const navigate = useNavigate();
|
||||
const [data, setData] = useState<ApiMonitorResponse | null>(null);
|
||||
const autoOpen = useApiMonitorOverlayStore((s) => s.autoOpen);
|
||||
const setAutoOpen = useApiMonitorOverlayStore((s) => s.setAutoOpen);
|
||||
|
||||
// One snapshot, not a poll: the live view is the monitor page.
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
void getApiMonitor()
|
||||
.then((next) => {
|
||||
if (!cancelled) setData(next);
|
||||
})
|
||||
.catch(() => undefined);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const active = data?.active_requests ?? 0;
|
||||
const recent = data?.entries.length ?? 0;
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-2">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
useSettingsDialogStore.getState().closeDialog();
|
||||
void navigate({ to: "/api-monitor" });
|
||||
}}
|
||||
className="flex w-full min-w-0 items-center gap-3 rounded-lg border border-border/70 bg-background px-4 py-3 text-left transition-colors hover:bg-muted/40 focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
|
||||
>
|
||||
<span className="relative flex size-8 shrink-0 items-center justify-center rounded-md border border-border/70 bg-muted/40">
|
||||
<HugeiconsIcon
|
||||
icon={ActivityIcon}
|
||||
strokeWidth={1.75}
|
||||
className="size-4"
|
||||
/>
|
||||
{active > 0 ? (
|
||||
<span className="absolute right-1 top-1 size-2 rounded-full bg-emerald-500" />
|
||||
) : null}
|
||||
</span>
|
||||
<span className="flex min-w-0 flex-col">
|
||||
<span className="text-sm font-semibold text-foreground">
|
||||
API monitor
|
||||
</span>
|
||||
<span className="truncate text-xs text-muted-foreground">
|
||||
{data == null
|
||||
? "Live requests, errors and token usage"
|
||||
: `${active.toLocaleString()} active · ${recent.toLocaleString()} recent · ${
|
||||
data.active_model ?? "no model loaded"
|
||||
}`}
|
||||
</span>
|
||||
</span>
|
||||
<HugeiconsIcon
|
||||
icon={ArrowRight02Icon}
|
||||
strokeWidth={1.75}
|
||||
className={cn("ml-auto size-4 shrink-0 text-muted-foreground")}
|
||||
/>
|
||||
</button>
|
||||
|
||||
{/* Where the panel's own "stop opening this" gets turned back on. */}
|
||||
<div className="flex items-center justify-between gap-3 rounded-lg px-1 py-1">
|
||||
<span className="flex min-w-0 flex-col">
|
||||
<span className="text-sm text-foreground">
|
||||
Show the floating monitor automatically
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Opens a small panel when API traffic arrives.
|
||||
</span>
|
||||
</span>
|
||||
<Switch
|
||||
checked={autoOpen}
|
||||
onCheckedChange={setAutoOpen}
|
||||
aria-label="Show the floating API monitor automatically"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ import { translate, useT } from "@/i18n";
|
|||
import { AnimatePresence, motion, useReducedMotion } from "motion/react";
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { fetchApiKeys, revokeApiKey, type ApiKey } from "../api/api-keys";
|
||||
import { ApiMonitorConsole } from "../components/api-monitor-console";
|
||||
import { MonitorLink } from "../components/monitor-link";
|
||||
import { ApiKeyRow } from "../components/api-key-row";
|
||||
import { CreateKeyForm } from "../components/create-key-form";
|
||||
import { ModelAutoSwitchSection } from "../components/model-auto-switch-section";
|
||||
|
|
@ -168,7 +168,7 @@ export function ApiKeysTab() {
|
|||
)}
|
||||
</section>
|
||||
|
||||
<ApiMonitorConsole />
|
||||
<MonitorLink />
|
||||
|
||||
<UsageExamples apiKey={revealed} />
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue