Keep native-leased GGUFs off the API mirror and date the monitor's first snapshot

Two identities the OpenAI-compatible auto-switch can never reach were being
treated as if it could.

A dropped or file-picked GGUF loads through a signed native-path lease, and
/api/inference/status reports model_identifier as null for it, so the checkpoint
the browser keys settings by is the bare file name the backend echoes back.
local_model_resolver._build_index keys a standalone GGUF by its on-disk path and
by its .gguf-stripped stem, so that name is never an index key: mirroring it
wrote a server override no load can read, and the monitor's "Settings applied on
API load" list then advertised it as live. The save gates on the lease token
rather than the name, since the label falls back to a plain string with no
suffix, and the one-time backfill, which has no token to read, goes by the
identity shape.

The floating monitor also wrote off every finished row of its first snapshot as
history. That snapshot is not taken at mount: a hidden tab issues no fetch at
all and an unreachable backend fails one, so the first API call of the session
can start and finish before it lands and never open the panel. Date the backlog
from when the poll stood up instead, on the server's own clock minus a browser
duration, so a browser that disagrees with the server cancels out rather than
replaying the whole ring buffer. A backend with no clock field keeps the old
behaviour. The decision moves into its own module so it can be driven without a
browser, which the overlay's .tsx dependency graph rules out.
This commit is contained in:
danielhanchen 2026-07-29 03:26:11 +00:00
commit c1f83a8c45
10 changed files with 352 additions and 34 deletions

View file

@ -6937,6 +6937,10 @@ async def get_api_monitor(current_subject: str = Depends(get_current_subject)):
operating_status = "idle"
return {
"status": operating_status,
# The clock every entry's started_at is on. The floating monitor dates its
# first snapshot against this instead of the browser's clock, which need not
# agree with ours over a tunnel or from a container.
"server_time": time.time(),
"active_model": active_model,
"context_length": _monitor_context_length(),
"active_requests": active_requests,

View file

@ -25,6 +25,13 @@ import {
useState,
} from "react";
import { isLifecycleEntry, lifecycleLabel } from "./lifecycle";
import {
type ApiMonitorWatch,
createWatch,
observeResponse,
rearmWatch,
startWatching,
} from "./new-traffic";
import { useApiMonitorOverlayStore } from "./overlay-store";
import { computeStats } from "./use-api-monitor";
@ -109,6 +116,10 @@ export function ApiMonitorOverlay(): ReactElement | null {
ReturnType<typeof getApiMonitor>
> | null>(null);
// What this session has already shown, and when it started watching.
const watchRef = useRef<ApiMonitorWatch>(createWatch(0));
const lastNewEntryAtRef = useRef(0);
// One loop for both jobs: panel contents while open, traffic watch while closed.
// Stands down on the full page, which polls for itself.
useEffect(() => {
@ -120,6 +131,11 @@ export function ApiMonitorOverlay(): ReactElement | null {
let timer: number | undefined;
const intervalMs = isOpen ? OPEN_POLL_MS : IDLE_POLL_MS;
// Anchor the watch here, not at mount: the first snapshot can arrive much
// later (a hidden tab skips its poll entirely, an unreachable backend fails
// one), and everything terminal in it would otherwise read as history.
startWatching(watchRef.current, performance.now());
function schedule(): void {
timer = window.setTimeout(poll, intervalMs);
}
@ -153,40 +169,15 @@ export function ApiMonitorOverlay(): ReactElement | null {
const entries = useMemo(() => data?.entries ?? [], [data]);
const stats = useMemo(() => computeStats(entries), [entries]);
// Ids already seen. A set, not "the newest id": finishing moves an entry to the
// front, so the head flips without any new traffic.
const seenIdsRef = useRef<Set<string>>(new Set());
// Seeded on the first response even when empty, so the first request of a fresh
// session is not mistaken for history.
const seededRef = useRef(false);
const lastNewEntryAtRef = useRef(0);
useEffect(() => {
if (data == null) {
return;
}
const ids = data.entries.map((entry) => entry.id);
if (!seededRef.current) {
seededRef.current = true;
// Seed finished requests only: one still running at the first snapshot started
// while Studio was loading, so it is unseen live traffic, not history.
seenIdsRef.current = new Set(
data.entries
.filter((entry) => entry.status !== "running")
.map((entry) => entry.id),
);
if (!data.entries.some((e) => e.via_api_key && e.status === "running")) {
return;
}
}
const seen = seenIdsRef.current;
// Only API-key traffic counts: Studio's own chat uses these same endpoints, and
// this panel is about serving other clients.
const hasNewTraffic = data.entries.some(
(entry) => entry.via_api_key && !seen.has(entry.id),
const hasNewTraffic = observeResponse(
watchRef.current,
data,
performance.now(),
);
// Re-seed each poll so the set stays bounded by the server's ring buffer.
seenIdsRef.current = new Set(ids);
if (!hasNewTraffic) {
return;
}
@ -203,10 +194,13 @@ export function ApiMonitorOverlay(): ReactElement | null {
open();
}, [data, autoOpen, suppressed, isOpen, open]);
// The backlog built up while the poll was stood down is not new traffic.
// The backlog built up while the poll was stood down is not new traffic. The
// watch re-anchors when the poll stands back up, not here, or the whole stay on
// the full page would read as unwatched and the panel would pop with rows the
// user has just read.
useEffect(() => {
if (onFullPage) {
seededRef.current = false;
rearmWatch(watchRef.current);
}
}, [onFullPage]);

View file

@ -0,0 +1,113 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
// Which rows of a monitor snapshot are traffic this session has not shown yet.
// Split out of the overlay so it can be driven without a browser.
import type { ApiMonitorEntry } from "@/features/chat/types/api";
export type WatchedEntry = Pick<
ApiMonitorEntry,
"id" | "status" | "via_api_key" | "started_at"
>;
export interface WatchedResponse {
entries: readonly WatchedEntry[];
// biome-ignore lint/style/useNamingConvention: API schema
server_time?: number | null;
}
export interface ApiMonitorWatch {
/** The first snapshot has been folded in and its backlog written off. */
seeded: boolean;
/** Ids already shown. A set, not "the newest id": finishing moves an entry to
* the front, so the head flips without any new traffic. */
seenIds: Set<string>;
/** performance.now() when this watch began; monotonic, so a client clock step
* mid-session cannot move it. */
watchStartedAt: number;
}
export function createWatch(nowMs: number): ApiMonitorWatch {
return { seeded: false, seenIds: new Set(), watchStartedAt: nowMs };
}
/**
* Re-anchor as the poll stands up, and only while still unseeded: the first
* snapshot can land long after the overlay mounted (a hidden tab issues no
* fetch, a backend still coming up fails one), and dating the backlog from
* mount would write that whole gap off as history.
*/
export function startWatching(watch: ApiMonitorWatch, nowMs: number): void {
if (!watch.seeded) {
watch.watchStartedAt = nowMs;
}
}
/** The full page took over; what it showed is not new traffic on the way back. */
export function rearmWatch(watch: ApiMonitorWatch): void {
watch.seeded = false;
}
/**
* When this watch began, on the server's clock.
*
* The server's own ``time.time()`` minus a browser *duration*, never minus a
* browser timestamp, so a browser clock that disagrees with the server's -- a
* Studio behind a tunnel, in a container, on a host that has not run NTP --
* cancels instead of skewing the answer. Null on a backend with no clock field,
* which keeps the old behaviour.
*/
function historyCutoff(
watch: ApiMonitorWatch,
response: WatchedResponse,
nowMs: number,
): number | null {
const serverTime = response.server_time;
if (typeof serverTime !== "number" || !Number.isFinite(serverTime)) {
return null;
}
return serverTime - Math.max(0, nowMs - watch.watchStartedAt) / 1000;
}
function isHistory(entry: WatchedEntry, cutoff: number | null): boolean {
// Still running at the first snapshot: it started while Studio was loading, so
// it is unseen live traffic.
if (entry.status === "running") {
return false;
}
if (cutoff == null || !Number.isFinite(entry.started_at)) {
return true;
}
// Finished before the first snapshot is not the same as started before we did:
// a call made while the tab was hidden is already terminal when the poll
// finally runs, and writing it off is how the panel misses the first request.
return entry.started_at <= cutoff;
}
/**
* Fold a snapshot in and report whether it holds API-key traffic not shown yet.
*/
export function observeResponse(
watch: ApiMonitorWatch,
response: WatchedResponse,
nowMs: number,
): boolean {
const { entries } = response;
if (!watch.seeded) {
watch.seeded = true;
const cutoff = historyCutoff(watch, response, nowMs);
watch.seenIds = new Set(
entries.filter((entry) => isHistory(entry, cutoff)).map((e) => e.id),
);
}
const seen = watch.seenIds;
// Only API-key traffic counts: Studio's own chat uses these same endpoints, and
// this panel is about serving other clients.
const hasNewTraffic = entries.some(
(entry) => entry.via_api_key && !seen.has(entry.id),
);
// Re-seed each poll so the set stays bounded by the server's ring buffer.
watch.seenIds = new Set(entries.map((entry) => entry.id));
return hasNewTraffic;
}

View file

@ -329,6 +329,10 @@ export interface ApiMonitorEntry {
export interface ApiMonitorResponse {
status: "idle" | "ready" | "generating";
// Server wall clock (seconds) when the snapshot was taken, so an entry's
// started_at can be dated without trusting the browser's clock to agree.
// Absent on a backend older than the field.
server_time?: number;
active_model?: string | null;
context_length?: number | null;
active_requests: number;

View file

@ -174,3 +174,17 @@ export function isOllamaLinkPath(modelId: string | null | undefined): boolean {
.split("/")
.some((segment) => OLLAMA_LINK_SEGMENTS.has(segment));
}
// A drag-dropped or file-picked GGUF is the API's second unreachable identity.
// /api/inference/status reports model_identifier as null for a lease-backed load
// (routes/inference.py withholds the host path), so the checkpoint the browser
// keys settings by is the bare file name the backend echoes back. _build_index
// keys a standalone GGUF by its on-disk path and by its .gguf-stripped stem, so
// that name is never an index key and no auto-switch load can read an override
// stored under it. Anything the API can load is keyed by a path or a repo id,
// both of which carry a separator.
const NATIVE_FILE_LABEL_RE = /^[^/\\]+\.gguf$/i;
export function isNativeFileLabel(modelId: string | null | undefined): boolean {
return modelId != null && NATIVE_FILE_LABEL_RE.test(modelId);
}

View file

@ -8,6 +8,7 @@
// API load uses app defaults, the exact bug the server-side map exists to fix.
import {
isNativeFileLabel,
isOllamaLinkPath,
normalizeGgufVariantIdentity,
normalizeModelIdentity,
@ -101,11 +102,13 @@ export async function backfillModelOverrides(): Promise<void> {
// .gguf has no quant to select between and is stored with a null variant, so it
// needs the extra test or its settings stay browser-only for good. An Ollama
// blob is GGUF but reached through a link dir the resolver skips, so it is not
// auto-switchable either.
// auto-switchable either, and a bare file name is a dropped/picked file's label,
// which the resolver never keys.
(entry) =>
(entry.ggufVariant != null ||
entry.modelId.toLowerCase().endsWith(".gguf")) &&
!isOllamaLinkPath(entry.modelId) &&
!isNativeFileLabel(entry.modelId) &&
!isDefaultConfig(entry.config),
);
if (local.length === 0) {

View file

@ -940,7 +940,15 @@ export function ModelConfigPage({
// two would permanently disagree with no way to tell which the next load used.
// Gated on auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and
// skips Ollama, so mirroring either would advertise a load that cannot happen.
if (!saveFailed && (target.apiLoadable ?? target.isGguf)) {
// A native-path lease is the same case: /status withholds model_identifier for a
// dropped or file-picked GGUF, so this id is only the file's display name, which
// the resolver never keys, and reopening the file needs a lease the API cannot
// mint. The token, not the name, decides: the fallback label carries no suffix.
if (
!saveFailed &&
(target.apiLoadable ?? target.isGguf) &&
!nativePathToken
) {
syncModelOverride(
configId,
target.ggufVariant,

View file

@ -10,6 +10,7 @@ import {
} from "@/features/hub/lib/model-identity";
export {
isNativeFileLabel,
isOllamaLinkPath,
normalizeGgufVariantIdentity,
normalizeModelIdentity,

View file

@ -0,0 +1,140 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import assert from "node:assert/strict";
import test from "node:test";
// The overlay is a .tsx pulling in motion, hugeicons and the router, so it cannot
// be imported here. Its new-traffic decision lives in a plain module for exactly
// that reason, and this drives the real one the overlay calls.
import {
type WatchedEntry,
type WatchedResponse,
createWatch,
observeResponse,
rearmWatch,
startWatching,
} from "../src/features/api-monitor/new-traffic.ts";
// The server's clock. Entry timestamps are its time.time(), so the tests keep them
// in those units and never mix in a browser instant.
const SERVER_NOW = 1_000_000;
// performance.now() when the poll stood up.
const WATCH_AT = 1_000;
function entry(
id: string,
status: WatchedEntry["status"],
startedAt: number,
viaApiKey = true,
): WatchedEntry {
// biome-ignore lint/style/useNamingConvention: API schema
return { id, status, via_api_key: viaApiKey, started_at: startedAt };
}
function snapshot(
entries: WatchedEntry[],
serverTime: number | null = SERVER_NOW,
): WatchedResponse {
// biome-ignore lint/style/useNamingConvention: API schema
return { entries, server_time: serverTime };
}
function watchFrom(startedAtMs: number) {
const watch = createWatch(0);
startWatching(watch, startedAtMs);
return watch;
}
test("a call that finished before the first snapshot arrived is new traffic", () => {
// The tab was hidden for 4s after the poll stood up, so poll() issued no fetch.
// The user's first curl ran 2s into that gap and was already done when the
// snapshot finally landed. Terminal, but not history.
const opened = observeResponse(
watchFrom(WATCH_AT),
snapshot([entry("apireq_new", "completed", SERVER_NOW - 2)]),
WATCH_AT + 4_000,
);
assert.equal(opened, true);
});
test("traffic from before the watch began stays history", () => {
const opened = observeResponse(
watchFrom(WATCH_AT),
snapshot([entry("apireq_old", "completed", SERVER_NOW - 90)]),
WATCH_AT + 4_000,
);
assert.equal(opened, false);
});
test("a request still running at the first snapshot is live traffic", () => {
const opened = observeResponse(
watchFrom(WATCH_AT),
snapshot([entry("apireq_live", "running", SERVER_NOW - 90)]),
WATCH_AT + 10,
);
assert.equal(opened, true);
});
test("a fresh id in a later snapshot opens the panel", () => {
const watch = watchFrom(WATCH_AT);
const backlog = [entry("apireq_old", "completed", SERVER_NOW - 90)];
assert.equal(observeResponse(watch, snapshot(backlog), WATCH_AT + 10), false);
const opened = observeResponse(
watch,
snapshot(
[entry("apireq_next", "completed", SERVER_NOW + 4), ...backlog],
SERVER_NOW + 5,
),
WATCH_AT + 5_010,
);
assert.equal(opened, true);
});
test("Studio's own chat never opens the panel", () => {
const opened = observeResponse(
watchFrom(WATCH_AT),
snapshot([entry("uireq", "completed", SERVER_NOW - 2, false)]),
WATCH_AT + 4_000,
);
assert.equal(opened, false);
});
test("a backend with no clock field keeps the old terminal-is-history seed", () => {
const opened = observeResponse(
watchFrom(WATCH_AT),
snapshot([entry("apireq_new", "completed", SERVER_NOW - 2)], null),
WATCH_AT + 4_000,
);
assert.equal(opened, false);
});
test("a browser clock disagreeing with the server's does not replay the backlog", () => {
// The cutoff is the server's own clock minus a browser DURATION, never minus a
// browser timestamp, so a browser whose wall clock is minutes off still dates
// the backlog correctly.
const opened = observeResponse(
watchFrom(WATCH_AT),
snapshot([
entry("apireq_a", "completed", SERVER_NOW - 300),
entry("apireq_b", "completed", SERVER_NOW - 120),
]),
WATCH_AT + 20,
);
assert.equal(opened, false);
});
test("coming back from the full page does not replay the rows it showed", () => {
const watch = watchFrom(WATCH_AT);
const backlog = [entry("apireq_old", "completed", SERVER_NOW - 90)];
observeResponse(watch, snapshot(backlog), WATCH_AT + 10);
// 60s on /api-monitor reading those rows, then back to chat.
rearmWatch(watch);
startWatching(watch, WATCH_AT + 60_000);
const opened = observeResponse(
watch,
snapshot(backlog, SERVER_NOW + 60),
WATCH_AT + 60_010,
);
assert.equal(opened, false);
});

View file

@ -953,11 +953,48 @@ def test_only_gguf_configs_are_mirrored_to_the_server():
loads safetensors models and must honour their config.
"""
src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split())
assert "if (!saveFailed && (target.apiLoadable ?? target.isGguf)) { syncModelOverride(" in src
assert (
"if ( !saveFailed && (target.apiLoadable ?? target.isGguf) && !nativePathToken ) "
"{ syncModelOverride(" in src
)
# The local save is not behind the same gate.
assert "if (remember) { saveFailed = !savePerModelConfig(" in src
def test_a_native_leased_gguf_is_not_mirrored_to_the_server():
"""A dropped or file-picked GGUF loads through a signed native-path lease, and
/api/inference/status reports model_identifier as null for it, so the checkpoint
the browser keys settings by is the bare file name the backend echoes back.
_build_index keys a standalone GGUF by its on-disk path and by its .gguf-stripped
stem, so that name is never an index key: mirroring it wrote an override no load
can read, which the monitor's applied-on-API-load list then advertised as live.
The live save gates on the lease token rather than the name, because the label
falls back to a plain string with no suffix when the host reports none.
"""
page = " ".join(_read("features/model-picker/components/model-config-page.tsx").split())
assert "&& !nativePathToken ) { syncModelOverride(" in page
assert (
"const nativePathToken = target.meta.nativePathToken ?? "
"(isActiveModel ? activeNativePathToken : null);" in page
), "the token this gate reads"
# The one-time backfill has no token to read, so it goes by the identity shape.
backfill = " ".join(_read("features/model-picker/api/migrate-model-overrides.ts").split())
assert "!isNativeFileLabel(entry.modelId) &&" in backfill
identity = " ".join(_read("features/hub/lib/model-identity.ts").split())
assert "export function isNativeFileLabel(" in identity
# A bare file name: no separator, and the .gguf the resolver's keys never carry.
assert "const NATIVE_FILE_LABEL_RE = /^[^/\\\\]+\\.gguf$/i;" in identity
backend = WORKDIR / "studio" / "backend"
inference = (backend / "routes" / "inference.py").read_text(encoding = "utf-8")
assert (
"model_identifier = None if _native_grant_backed else _model_id" in inference
), "why the checkpoint is only a display name"
models = (backend / "routes" / "models.py").read_text(encoding = "utf-8")
assert "display_name = gguf_file.stem," in models, "why the name is never an index key"
def test_evicted_local_configs_drop_their_server_overrides():
"""savePerModelConfig evicts older models when the map exceeds its budget.
Those models keep a server override that API loads still apply, with nothing