refactor(llm): inclusive total + non-overlapping breakdown for Usage

Final shape after considering ecosystem conventions:

  inputTokens             — inclusive total (matches AI SDK / OpenAI / LangChain)
  outputTokens            — inclusive total (includes reasoning)
  nonCachedInputTokens    — breakdown: fresh prompt
  cacheReadInputTokens    — breakdown: cache hit
  cacheWriteInputTokens   — breakdown: cache write
  reasoningTokens         — subset of outputTokens

Invariant:
  nonCached + cacheRead + cacheWrite = inputTokens
  reasoningTokens <= outputTokens

Why this shape:

- `inputTokens` keeps its AI-SDK / OpenAI semantics, so a reader from any
  major ecosystem sees the number they expect.
- The non-overlapping breakdown fields are populated alongside the
  inclusive totals — consumers read whichever they need without
  subtracting. This eliminates the underflow bug class (opencode#26620)
  structurally without diverging on naming.
- Aligns with the AI SDK v3 spec proposal (vercel/ai#9921), which adds
  exactly this kind of non-overlapping breakdown to address the active
  ecosystem bugs around cache token double-counting and underflow
  (pydantic-ai#4364, langfuse#12306/#11979, vercel/ai#8349,
  langchain#32818, langchainjs#10249).

Mappers:

- OpenAI Chat / Responses / Bedrock: provider reports inclusive totals
  natively; mapper derives `nonCachedInputTokens` via
  `ProviderShared.subtractTokens`.
- Gemini: `promptTokenCount` is inclusive; `candidatesTokenCount` is
  *exclusive* of `thoughtsTokenCount`, so mapper sums those to produce
  the inclusive `outputTokens`. Only computes the total when the visible
  component is reported (avoids fabricating an inclusive number from a
  partial breakdown).
- Anthropic: `input_tokens` is *non-cached* natively; mapper sums it with
  cache reads/writes to produce the inclusive `inputTokens`.
  `output_tokens` is inclusive (Anthropic doesn't break thinking out, so
  `reasoningTokens` stays undefined).

Added a `visibleOutputTokens` getter (clamped `outputTokens - reasoningTokens`)
as the one safe escape hatch for consumers wanting the non-reasoning view.

Added `ProviderShared.sumTokens` to derive an inclusive total from a
non-overlapping breakdown, returning `undefined` when every input is
undefined (so we don't fabricate a 0).
This commit is contained in:
Kit Langton 2026-05-10 20:39:22 -04:00
commit d4ff331052
12 changed files with 173 additions and 115 deletions

View file

@ -110,10 +110,11 @@ describe("Anthropic Messages route", () => {
expect(response.text).toBe("Hello!")
expect(response.reasoning).toBe("thinking")
expect(response.usage).toMatchObject({
inputTokens: 5,
inputTokens: 6,
outputTokens: 2,
nonCachedInputTokens: 5,
cacheReadInputTokens: 1,
totalTokens: 7,
totalTokens: 8,
})
expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
providerMetadata: { anthropic: { signature: "sig_1" } },
@ -152,7 +153,13 @@ describe("Anthropic Messages route", () => {
{
type: "request-finish",
reason: "tool-calls",
usage: new Usage({ inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } }),
usage: new Usage({
inputTokens: 5,
outputTokens: 1,
nonCachedInputTokens: 5,
totalTokens: 6,
native: { input_tokens: 5, output_tokens: 1 },
}),
},
])
}),

View file

@ -197,10 +197,11 @@ describe("Gemini route", () => {
expect(response.text).toBe("Hello!")
expect(response.reasoning).toBe("thinking")
expect(response.usage).toMatchObject({
inputTokens: 4,
outputTokens: 2,
reasoningTokens: 1,
inputTokens: 5,
outputTokens: 3,
nonCachedInputTokens: 4,
cacheReadInputTokens: 1,
reasoningTokens: 1,
totalTokens: 7,
})
expect(response.events).toEqual([
@ -211,10 +212,11 @@ describe("Gemini route", () => {
type: "request-finish",
reason: "stop",
usage: new Usage({
inputTokens: 4,
outputTokens: 2,
reasoningTokens: 1,
inputTokens: 5,
outputTokens: 3,
nonCachedInputTokens: 4,
cacheReadInputTokens: 1,
reasoningTokens: 1,
totalTokens: 7,
native: {
promptTokenCount: 5,
@ -260,6 +262,7 @@ describe("Gemini route", () => {
usage: new Usage({
inputTokens: 5,
outputTokens: 1,
nonCachedInputTokens: 5,
totalTokens: 6,
native: { promptTokenCount: 5, candidatesTokenCount: 1 },
}),

View file

@ -231,10 +231,11 @@ describe("OpenAI Chat route", () => {
type: "request-finish",
reason: "stop",
usage: new Usage({
inputTokens: 4,
inputTokens: 5,
outputTokens: 2,
reasoningTokens: 0,
nonCachedInputTokens: 4,
cacheReadInputTokens: 1,
reasoningTokens: 0,
totalTokens: 7,
native: {
prompt_tokens: 5,

View file

@ -343,10 +343,11 @@ describe("OpenAI Responses route", () => {
reason: "stop",
providerMetadata: { openai: { responseId: "resp_1", serviceTier: "default" } },
usage: new Usage({
inputTokens: 4,
inputTokens: 5,
outputTokens: 2,
reasoningTokens: 0,
nonCachedInputTokens: 4,
cacheReadInputTokens: 1,
reasoningTokens: 0,
totalTokens: 7,
native: {
input_tokens: 5,
@ -411,7 +412,13 @@ describe("OpenAI Responses route", () => {
{
type: "request-finish",
reason: "tool-calls",
usage: new Usage({ inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } }),
usage: new Usage({
inputTokens: 5,
outputTokens: 1,
nonCachedInputTokens: 5,
totalTokens: 6,
native: { input_tokens: 5, output_tokens: 1 },
}),
},
])
}),

View file

@ -50,7 +50,7 @@ describe("llm schema", () => {
})
})
describe("LLM.Usage additive contract", () => {
describe("LLM.Usage", () => {
test("subtractTokens clamps non-sensical breakdowns to zero", () => {
// Defense against a provider reporting cached_tokens > prompt_tokens or
// reasoning_tokens > completion_tokens — the negative would otherwise
@ -62,15 +62,17 @@ describe("LLM.Usage additive contract", () => {
expect(ProviderShared.subtractTokens(undefined, undefined)).toBeUndefined()
})
test("totalInputTokens sums every input-side category", () => {
expect(new Usage({ inputTokens: 10, cacheReadInputTokens: 3, cacheWriteInputTokens: 2 }).totalInputTokens).toBe(15)
expect(new Usage({ inputTokens: 10 }).totalInputTokens).toBe(10)
expect(new Usage({}).totalInputTokens).toBe(0)
test("sumTokens returns undefined only when every input is undefined", () => {
expect(ProviderShared.sumTokens(1, 2, 3)).toBe(6)
expect(ProviderShared.sumTokens(1, undefined, 3)).toBe(4)
expect(ProviderShared.sumTokens(undefined, undefined, undefined)).toBeUndefined()
expect(ProviderShared.sumTokens()).toBeUndefined()
})
test("totalOutputTokens sums every output-side category", () => {
expect(new Usage({ outputTokens: 7, reasoningTokens: 4 }).totalOutputTokens).toBe(11)
expect(new Usage({ outputTokens: 7 }).totalOutputTokens).toBe(7)
expect(new Usage({}).totalOutputTokens).toBe(0)
test("visibleOutputTokens clamps reasoning > output to zero", () => {
expect(new Usage({ outputTokens: 10, reasoningTokens: 4 }).visibleOutputTokens).toBe(6)
expect(new Usage({ outputTokens: 10 }).visibleOutputTokens).toBe(10)
expect(new Usage({ outputTokens: 4, reasoningTokens: 10 }).visibleOutputTokens).toBe(0)
expect(new Usage({}).visibleOutputTokens).toBe(0)
})
})