Studio: tighten cache-stats comments

This commit is contained in:
Daniel Han 2026-05-23 12:44:20 +00:00 committed by danielhanchen
commit b2644661ca
6 changed files with 15 additions and 57 deletions

View file

@ -45,14 +45,9 @@ export const MessageTiming: FC<{
}
| undefined;
const st = custom?.serverTimings;
// Cache-hit / cache-write counts. llama-server reports hits on
// timings.cache_n; external providers (Anthropic / OpenAI Responses /
// Gemini) report them on the include_usage envelope that the adapter
// normalizes into custom.contextUsage. Prefer the local-runtime value
// when present (so llama.cpp keeps populating the badge mid-stream)
// and fall back to the external envelope otherwise.
// Prefer llama-server timings, fall back to external usage envelope.
const cacheHits = (st?.cache_n ?? 0) || (custom?.contextUsage?.cachedTokens ?? 0);
// Anthropic-only: tokens written into the prompt cache on this turn.
// Anthropic-only cache-write count.
const cacheWrites = custom?.contextUsage?.cacheWriteTokens ?? 0;
// Guard unphysical tok/s: llama.cpp emits predicted_ms=0 on no-op

View file

@ -70,16 +70,9 @@ interface ServerUsage {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
/**
* External providers (Anthropic / OpenAI Responses / Gemini) surface
* prompt-cache accounting on the same `usage` envelope via
* `_build_usage_chunk` in `studio/backend/core/inference/external_provider.py`.
* `prompt_tokens_details.cached_tokens` is the normalised cache-read count
* present for every provider that supports it; `cache_creation_input_tokens`
* / `cache_read_input_tokens` are the Anthropic-native keys (cache_read
* mirrors `prompt_tokens_details.cached_tokens`; cache_creation is
* Anthropic-only and billed at the cache-write premium).
*/
// External prompt-cache fields from `_build_usage_chunk` in
// studio/backend/core/inference/external_provider.py. cache_read mirrors
// prompt_tokens_details.cached_tokens; cache_creation is Anthropic-only.
prompt_tokens_details?: {
cached_tokens?: number;
};
@ -1896,24 +1889,14 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
const finalTokPerSec = meta?.timings?.predicted_per_second;
const serverPromptEvalTime = meta?.timings?.prompt_ms;
// Cache-hit count. llama-server reports it on `timings.cache_n`;
// external providers (Anthropic, OpenAI Responses, Gemini) report
// it on `usage.prompt_tokens_details.cached_tokens` -- see the
// `_build_usage_chunk` helper in
// studio/backend/core/inference/external_provider.py. Prefer the
// local-runtime value when it is present (so llama.cpp keeps
// populating the bar mid-stream) and fall back to the external
// usage envelope otherwise. cache_read_input_tokens is Anthropic's
// native key for the same value; read it as a last resort for
// providers that only emit the Anthropic shape.
// Cache-hit count: prefer llama-server timings, fall back to the
// external-provider usage envelope (Anthropic-native key last).
const cachedTokens =
meta?.timings?.cache_n ??
meta?.usage?.prompt_tokens_details?.cached_tokens ??
meta?.usage?.cache_read_input_tokens ??
0;
// Anthropic-only: tokens written into the prompt cache on this
// turn, billed at the cache-write premium. Surfaced separately so
// users can tell a cache miss from a cache hit.
// Anthropic-only cache-write count (billed at the write premium).
const cacheWriteTokens = meta?.usage?.cache_creation_input_tokens ?? 0;
// Update context usage in store if we got valid server data

View file

@ -1494,12 +1494,7 @@ export function ChatPage(): ReactElement {
{view.mode === "single" && contextUsage ? (
<ContextUsageBar
used={contextUsage.totalTokens}
// ggufContextLength is the local llama-server's KV-cache size;
// external providers (Anthropic / OpenAI Responses / Gemini)
// don't expose a stable per-model context window through the
// picker, so it is null in that mode. Pass it as-is -- the bar
// drops the "/ total" ratio + percentage when total is absent
// and still renders the per-turn counters + cache stats.
// null on external providers; the bar handles that.
total={ggufContextLength}
cached={contextUsage.cachedTokens}
cacheWrites={contextUsage.cacheWriteTokens}

View file

@ -28,21 +28,11 @@ function getSeverityColor(percent: number): {
export const ContextUsageBar: FC<{
used: number;
/**
* Context window size. Optional because external providers don't expose a
* stable per-model limit through the chat picker -- the local llama-server
* path is the only one that populates ggufContextLength. When omitted, the
* bar drops the "/ total" ratio + percentage bar and just shows the token
* counts (so cache hits / writes from Anthropic / OpenAI Responses still
* land in the tooltip).
*/
// Optional: only local llama-server knows the context window. When absent
// the bar shows token counts without the "/ total" ratio.
total?: number | null;
cached?: number;
/**
* Anthropic-only cache-write count (tokens written into the prompt cache
* on this turn, billed at the cache-write premium). Shown as a separate
* tooltip line so users can tell a cache miss from a cache hit.
*/
// Anthropic-only cache-write count (billed at the write premium).
cacheWrites?: number;
promptTokens?: number;
completionTokens?: number;

View file

@ -831,11 +831,8 @@ function useStudioRuntimeAdapters(): StudioRuntimeAdapters {
}
| undefined;
const store = useChatRuntimeStore.getState();
// External-provider threads have ggufContextLength === null because
// external picker selections don't carry a context-window number.
// Restore the persisted usage as long as it belongs to the active
// checkpoint; when a local GGUF context window IS known, also keep
// the original sanity check that the saved total fits inside it.
// Only enforce the fits-in-window check when a local GGUF window
// is known; external providers have ggufContextLength === null.
const withinLocalLimit =
!store.ggufContextLength ||
(savedUsage?.totalTokens ?? 0) <= store.ggufContextLength;

View file

@ -291,9 +291,7 @@ type ChatRuntimeStore = {
completionTokens: number;
totalTokens: number;
cachedTokens: number;
// Anthropic-only cache-write count from
// `usage.cache_creation_input_tokens`. Optional so older persisted
// entries from llama-server / pre-cache-stats builds keep loading.
// Anthropic-only; optional so pre-cache-stats persisted entries load.
cacheWriteTokens?: number;
} | null;
modelLoading: boolean;