refactor(llm): inclusive total + non-overlapping breakdown for Usage
Final shape after considering ecosystem conventions: inputTokens — inclusive total (matches AI SDK / OpenAI / LangChain) outputTokens — inclusive total (includes reasoning) nonCachedInputTokens — breakdown: fresh prompt cacheReadInputTokens — breakdown: cache hit cacheWriteInputTokens — breakdown: cache write reasoningTokens — subset of outputTokens Invariant: nonCached + cacheRead + cacheWrite = inputTokens reasoningTokens <= outputTokens Why this shape: - `inputTokens` keeps its AI-SDK / OpenAI semantics, so a reader from any major ecosystem sees the number they expect. - The non-overlapping breakdown fields are populated alongside the inclusive totals — consumers read whichever they need without subtracting. This eliminates the underflow bug class (opencode#26620) structurally without diverging on naming. - Aligns with the AI SDK v3 spec proposal (vercel/ai#9921), which adds exactly this kind of non-overlapping breakdown to address the active ecosystem bugs around cache token double-counting and underflow (pydantic-ai#4364, langfuse#12306/#11979, vercel/ai#8349, langchain#32818, langchainjs#10249). Mappers: - OpenAI Chat / Responses / Bedrock: provider reports inclusive totals natively; mapper derives `nonCachedInputTokens` via `ProviderShared.subtractTokens`. - Gemini: `promptTokenCount` is inclusive; `candidatesTokenCount` is *exclusive* of `thoughtsTokenCount`, so mapper sums those to produce the inclusive `outputTokens`. Only computes the total when the visible component is reported (avoids fabricating an inclusive number from a partial breakdown). - Anthropic: `input_tokens` is *non-cached* natively; mapper sums it with cache reads/writes to produce the inclusive `inputTokens`. `output_tokens` is inclusive (Anthropic doesn't break thinking out, so `reasoningTokens` stays undefined). Added a `visibleOutputTokens` getter (clamped `outputTokens - reasoningTokens`) as the one safe escape hatch for consumers wanting the non-reasoning view. Added `ProviderShared.sumTokens` to derive an inclusive total from a non-overlapping breakdown, returning `undefined` when every input is undefined (so we don't fabricate a 0).
This commit is contained in:
parent
f5d199db62
commit
d4ff331052
12 changed files with 173 additions and 115 deletions
|
|
@ -110,10 +110,11 @@ describe("Anthropic Messages route", () => {
|
|||
expect(response.text).toBe("Hello!")
|
||||
expect(response.reasoning).toBe("thinking")
|
||||
expect(response.usage).toMatchObject({
|
||||
inputTokens: 5,
|
||||
inputTokens: 6,
|
||||
outputTokens: 2,
|
||||
nonCachedInputTokens: 5,
|
||||
cacheReadInputTokens: 1,
|
||||
totalTokens: 7,
|
||||
totalTokens: 8,
|
||||
})
|
||||
expect(response.events.find((event) => event.type === "reasoning-end")).toMatchObject({
|
||||
providerMetadata: { anthropic: { signature: "sig_1" } },
|
||||
|
|
@ -152,7 +153,13 @@ describe("Anthropic Messages route", () => {
|
|||
{
|
||||
type: "request-finish",
|
||||
reason: "tool-calls",
|
||||
usage: new Usage({ inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } }),
|
||||
usage: new Usage({
|
||||
inputTokens: 5,
|
||||
outputTokens: 1,
|
||||
nonCachedInputTokens: 5,
|
||||
totalTokens: 6,
|
||||
native: { input_tokens: 5, output_tokens: 1 },
|
||||
}),
|
||||
},
|
||||
])
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -197,10 +197,11 @@ describe("Gemini route", () => {
|
|||
expect(response.text).toBe("Hello!")
|
||||
expect(response.reasoning).toBe("thinking")
|
||||
expect(response.usage).toMatchObject({
|
||||
inputTokens: 4,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 1,
|
||||
inputTokens: 5,
|
||||
outputTokens: 3,
|
||||
nonCachedInputTokens: 4,
|
||||
cacheReadInputTokens: 1,
|
||||
reasoningTokens: 1,
|
||||
totalTokens: 7,
|
||||
})
|
||||
expect(response.events).toEqual([
|
||||
|
|
@ -211,10 +212,11 @@ describe("Gemini route", () => {
|
|||
type: "request-finish",
|
||||
reason: "stop",
|
||||
usage: new Usage({
|
||||
inputTokens: 4,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 1,
|
||||
inputTokens: 5,
|
||||
outputTokens: 3,
|
||||
nonCachedInputTokens: 4,
|
||||
cacheReadInputTokens: 1,
|
||||
reasoningTokens: 1,
|
||||
totalTokens: 7,
|
||||
native: {
|
||||
promptTokenCount: 5,
|
||||
|
|
@ -260,6 +262,7 @@ describe("Gemini route", () => {
|
|||
usage: new Usage({
|
||||
inputTokens: 5,
|
||||
outputTokens: 1,
|
||||
nonCachedInputTokens: 5,
|
||||
totalTokens: 6,
|
||||
native: { promptTokenCount: 5, candidatesTokenCount: 1 },
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -231,10 +231,11 @@ describe("OpenAI Chat route", () => {
|
|||
type: "request-finish",
|
||||
reason: "stop",
|
||||
usage: new Usage({
|
||||
inputTokens: 4,
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 0,
|
||||
nonCachedInputTokens: 4,
|
||||
cacheReadInputTokens: 1,
|
||||
reasoningTokens: 0,
|
||||
totalTokens: 7,
|
||||
native: {
|
||||
prompt_tokens: 5,
|
||||
|
|
|
|||
|
|
@ -343,10 +343,11 @@ describe("OpenAI Responses route", () => {
|
|||
reason: "stop",
|
||||
providerMetadata: { openai: { responseId: "resp_1", serviceTier: "default" } },
|
||||
usage: new Usage({
|
||||
inputTokens: 4,
|
||||
inputTokens: 5,
|
||||
outputTokens: 2,
|
||||
reasoningTokens: 0,
|
||||
nonCachedInputTokens: 4,
|
||||
cacheReadInputTokens: 1,
|
||||
reasoningTokens: 0,
|
||||
totalTokens: 7,
|
||||
native: {
|
||||
input_tokens: 5,
|
||||
|
|
@ -411,7 +412,13 @@ describe("OpenAI Responses route", () => {
|
|||
{
|
||||
type: "request-finish",
|
||||
reason: "tool-calls",
|
||||
usage: new Usage({ inputTokens: 5, outputTokens: 1, totalTokens: 6, native: { input_tokens: 5, output_tokens: 1 } }),
|
||||
usage: new Usage({
|
||||
inputTokens: 5,
|
||||
outputTokens: 1,
|
||||
nonCachedInputTokens: 5,
|
||||
totalTokens: 6,
|
||||
native: { input_tokens: 5, output_tokens: 1 },
|
||||
}),
|
||||
},
|
||||
])
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ describe("llm schema", () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe("LLM.Usage additive contract", () => {
|
||||
describe("LLM.Usage", () => {
|
||||
test("subtractTokens clamps non-sensical breakdowns to zero", () => {
|
||||
// Defense against a provider reporting cached_tokens > prompt_tokens or
|
||||
// reasoning_tokens > completion_tokens — the negative would otherwise
|
||||
|
|
@ -62,15 +62,17 @@ describe("LLM.Usage additive contract", () => {
|
|||
expect(ProviderShared.subtractTokens(undefined, undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
test("totalInputTokens sums every input-side category", () => {
|
||||
expect(new Usage({ inputTokens: 10, cacheReadInputTokens: 3, cacheWriteInputTokens: 2 }).totalInputTokens).toBe(15)
|
||||
expect(new Usage({ inputTokens: 10 }).totalInputTokens).toBe(10)
|
||||
expect(new Usage({}).totalInputTokens).toBe(0)
|
||||
test("sumTokens returns undefined only when every input is undefined", () => {
|
||||
expect(ProviderShared.sumTokens(1, 2, 3)).toBe(6)
|
||||
expect(ProviderShared.sumTokens(1, undefined, 3)).toBe(4)
|
||||
expect(ProviderShared.sumTokens(undefined, undefined, undefined)).toBeUndefined()
|
||||
expect(ProviderShared.sumTokens()).toBeUndefined()
|
||||
})
|
||||
|
||||
test("totalOutputTokens sums every output-side category", () => {
|
||||
expect(new Usage({ outputTokens: 7, reasoningTokens: 4 }).totalOutputTokens).toBe(11)
|
||||
expect(new Usage({ outputTokens: 7 }).totalOutputTokens).toBe(7)
|
||||
expect(new Usage({}).totalOutputTokens).toBe(0)
|
||||
test("visibleOutputTokens clamps reasoning > output to zero", () => {
|
||||
expect(new Usage({ outputTokens: 10, reasoningTokens: 4 }).visibleOutputTokens).toBe(6)
|
||||
expect(new Usage({ outputTokens: 10 }).visibleOutputTokens).toBe(10)
|
||||
expect(new Usage({ outputTokens: 4, reasoningTokens: 10 }).visibleOutputTokens).toBe(0)
|
||||
expect(new Usage({}).visibleOutputTokens).toBe(0)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue