Studio: tighten cache-stats comments
This commit is contained in:
parent
79da85ba7f
commit
b2644661ca
6 changed files with 15 additions and 57 deletions
|
|
@ -45,14 +45,9 @@ export const MessageTiming: FC<{
|
|||
}
|
||||
| undefined;
|
||||
const st = custom?.serverTimings;
|
||||
// Cache-hit / cache-write counts. llama-server reports hits on
|
||||
// timings.cache_n; external providers (Anthropic / OpenAI Responses /
|
||||
// Gemini) report them on the include_usage envelope that the adapter
|
||||
// normalizes into custom.contextUsage. Prefer the local-runtime value
|
||||
// when present (so llama.cpp keeps populating the badge mid-stream)
|
||||
// and fall back to the external envelope otherwise.
|
||||
// Prefer llama-server timings, fall back to external usage envelope.
|
||||
const cacheHits = (st?.cache_n ?? 0) || (custom?.contextUsage?.cachedTokens ?? 0);
|
||||
// Anthropic-only: tokens written into the prompt cache on this turn.
|
||||
// Anthropic-only cache-write count.
|
||||
const cacheWrites = custom?.contextUsage?.cacheWriteTokens ?? 0;
|
||||
|
||||
// Guard unphysical tok/s: llama.cpp emits predicted_ms=0 on no-op
|
||||
|
|
|
|||
|
|
@ -70,16 +70,9 @@ interface ServerUsage {
|
|||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
/**
|
||||
* External providers (Anthropic / OpenAI Responses / Gemini) surface
|
||||
* prompt-cache accounting on the same `usage` envelope via
|
||||
* `_build_usage_chunk` in `studio/backend/core/inference/external_provider.py`.
|
||||
* `prompt_tokens_details.cached_tokens` is the normalised cache-read count
|
||||
* present for every provider that supports it; `cache_creation_input_tokens`
|
||||
* / `cache_read_input_tokens` are the Anthropic-native keys (cache_read
|
||||
* mirrors `prompt_tokens_details.cached_tokens`; cache_creation is
|
||||
* Anthropic-only and billed at the cache-write premium).
|
||||
*/
|
||||
// External prompt-cache fields from `_build_usage_chunk` in
|
||||
// studio/backend/core/inference/external_provider.py. cache_read mirrors
|
||||
// prompt_tokens_details.cached_tokens; cache_creation is Anthropic-only.
|
||||
prompt_tokens_details?: {
|
||||
cached_tokens?: number;
|
||||
};
|
||||
|
|
@ -1896,24 +1889,14 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
const finalTokPerSec = meta?.timings?.predicted_per_second;
|
||||
const serverPromptEvalTime = meta?.timings?.prompt_ms;
|
||||
|
||||
// Cache-hit count. llama-server reports it on `timings.cache_n`;
|
||||
// external providers (Anthropic, OpenAI Responses, Gemini) report
|
||||
// it on `usage.prompt_tokens_details.cached_tokens` -- see the
|
||||
// `_build_usage_chunk` helper in
|
||||
// studio/backend/core/inference/external_provider.py. Prefer the
|
||||
// local-runtime value when it is present (so llama.cpp keeps
|
||||
// populating the bar mid-stream) and fall back to the external
|
||||
// usage envelope otherwise. cache_read_input_tokens is Anthropic's
|
||||
// native key for the same value; read it as a last resort for
|
||||
// providers that only emit the Anthropic shape.
|
||||
// Cache-hit count: prefer llama-server timings, fall back to the
|
||||
// external-provider usage envelope (Anthropic-native key last).
|
||||
const cachedTokens =
|
||||
meta?.timings?.cache_n ??
|
||||
meta?.usage?.prompt_tokens_details?.cached_tokens ??
|
||||
meta?.usage?.cache_read_input_tokens ??
|
||||
0;
|
||||
// Anthropic-only: tokens written into the prompt cache on this
|
||||
// turn, billed at the cache-write premium. Surfaced separately so
|
||||
// users can tell a cache miss from a cache hit.
|
||||
// Anthropic-only cache-write count (billed at the write premium).
|
||||
const cacheWriteTokens = meta?.usage?.cache_creation_input_tokens ?? 0;
|
||||
|
||||
// Update context usage in store if we got valid server data
|
||||
|
|
|
|||
|
|
@ -1494,12 +1494,7 @@ export function ChatPage(): ReactElement {
|
|||
{view.mode === "single" && contextUsage ? (
|
||||
<ContextUsageBar
|
||||
used={contextUsage.totalTokens}
|
||||
// ggufContextLength is the local llama-server's KV-cache size;
|
||||
// external providers (Anthropic / OpenAI Responses / Gemini)
|
||||
// don't expose a stable per-model context window through the
|
||||
// picker, so it is null in that mode. Pass it as-is -- the bar
|
||||
// drops the "/ total" ratio + percentage when total is absent
|
||||
// and still renders the per-turn counters + cache stats.
|
||||
// null on external providers; the bar handles that.
|
||||
total={ggufContextLength}
|
||||
cached={contextUsage.cachedTokens}
|
||||
cacheWrites={contextUsage.cacheWriteTokens}
|
||||
|
|
|
|||
|
|
@ -28,21 +28,11 @@ function getSeverityColor(percent: number): {
|
|||
|
||||
export const ContextUsageBar: FC<{
|
||||
used: number;
|
||||
/**
|
||||
* Context window size. Optional because external providers don't expose a
|
||||
* stable per-model limit through the chat picker -- the local llama-server
|
||||
* path is the only one that populates ggufContextLength. When omitted, the
|
||||
* bar drops the "/ total" ratio + percentage bar and just shows the token
|
||||
* counts (so cache hits / writes from Anthropic / OpenAI Responses still
|
||||
* land in the tooltip).
|
||||
*/
|
||||
// Optional: only local llama-server knows the context window. When absent
|
||||
// the bar shows token counts without the "/ total" ratio.
|
||||
total?: number | null;
|
||||
cached?: number;
|
||||
/**
|
||||
* Anthropic-only cache-write count (tokens written into the prompt cache
|
||||
* on this turn, billed at the cache-write premium). Shown as a separate
|
||||
* tooltip line so users can tell a cache miss from a cache hit.
|
||||
*/
|
||||
// Anthropic-only cache-write count (billed at the write premium).
|
||||
cacheWrites?: number;
|
||||
promptTokens?: number;
|
||||
completionTokens?: number;
|
||||
|
|
|
|||
|
|
@ -831,11 +831,8 @@ function useStudioRuntimeAdapters(): StudioRuntimeAdapters {
|
|||
}
|
||||
| undefined;
|
||||
const store = useChatRuntimeStore.getState();
|
||||
// External-provider threads have ggufContextLength === null because
|
||||
// external picker selections don't carry a context-window number.
|
||||
// Restore the persisted usage as long as it belongs to the active
|
||||
// checkpoint; when a local GGUF context window IS known, also keep
|
||||
// the original sanity check that the saved total fits inside it.
|
||||
// Only enforce the fits-in-window check when a local GGUF window
|
||||
// is known; external providers have ggufContextLength === null.
|
||||
const withinLocalLimit =
|
||||
!store.ggufContextLength ||
|
||||
(savedUsage?.totalTokens ?? 0) <= store.ggufContextLength;
|
||||
|
|
|
|||
|
|
@ -291,9 +291,7 @@ type ChatRuntimeStore = {
|
|||
completionTokens: number;
|
||||
totalTokens: number;
|
||||
cachedTokens: number;
|
||||
// Anthropic-only cache-write count from
|
||||
// `usage.cache_creation_input_tokens`. Optional so older persisted
|
||||
// entries from llama-server / pre-cache-stats builds keep loading.
|
||||
// Anthropic-only; optional so pre-cache-stats persisted entries load.
|
||||
cacheWriteTokens?: number;
|
||||
} | null;
|
||||
modelLoading: boolean;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue