From a214ac39deae415353dacc143207d1dd9e7bdff3 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Fri, 31 Jul 2026 17:52:32 -0500 Subject: [PATCH] refactor(ai): unify prompt cache configuration --- packages/ai/example/tutorial.ts | 9 +++--- packages/ai/src/cache-policy.ts | 18 +++++++---- packages/ai/src/protocols/open-responses.ts | 3 +- packages/ai/src/providers/openrouter.ts | 4 +-- packages/ai/src/schema/messages.ts | 3 -- packages/ai/src/schema/options.ts | 32 +++++++++++-------- packages/ai/test/cache-policy.test.ts | 20 ++++++------ .../test/provider-options/cloudflare.types.ts | 4 +-- .../openai-responses-cache.recorded.test.ts | 2 +- .../ai/test/provider/openai-responses.test.ts | 4 +-- packages/ai/test/provider/openrouter.test.ts | 6 ++-- packages/core/src/session/compaction.ts | 2 +- packages/core/src/session/generate-node.ts | 2 +- packages/core/src/session/model-request.ts | 2 +- packages/core/src/session/title.ts | 2 +- packages/core/test/session-compaction.test.ts | 2 +- packages/core/test/session-generate.test.ts | 2 +- packages/core/test/session-runner.test.ts | 4 +-- packages/core/test/session-title.test.ts | 2 +- 19 files changed, 64 insertions(+), 59 deletions(-) diff --git a/packages/ai/example/tutorial.ts b/packages/ai/example/tutorial.ts index 3924a57dd2..61ba0f0621 100644 --- a/packages/ai/example/tutorial.ts +++ b/packages/ai/example/tutorial.ts @@ -33,9 +33,10 @@ const model = OpenAI.configure({ // // - `generation`: common controls such as max tokens, temperature, topP/topK, // penalties, seed, and stop sequences. +// - `cache`: provider-neutral prompt caching behavior and cache affinity. // - `providerOptions`: namespaced provider-native behavior. For example, -// OpenAI cache keys and store behavior, Anthropic thinking, Gemini thinking -// config, or OpenRouter routing/reasoning. +// OpenAI store behavior, Anthropic thinking, Gemini thinking config, or +// OpenRouter routing/reasoning. // - `http`: last-resort serializable overlays for final request body, headers, // and query params. Prefer typed `providerOptions` when a field is stable. // @@ -45,9 +46,7 @@ const request = LLM.request({ system: "You are concise and practical.", prompt: "Tell me a joke", generation: { maxTokens: 80, temperature: 0.7 }, - providerOptions: { - openai: { promptCacheKey: "tutorial-joke" }, - }, + cache: { mode: "auto", key: "tutorial-joke" }, }) // 3. `generate` sends the request and collects the event stream into one diff --git a/packages/ai/src/cache-policy.ts b/packages/ai/src/cache-policy.ts index 1e314ea68a..1e1f08d572 100644 --- a/packages/ai/src/cache-policy.ts +++ b/packages/ai/src/cache-policy.ts @@ -10,16 +10,16 @@ // // Manual `cache: CacheHint` placements on individual parts are preserved and // count against the four-breakpoint budget; auto only fills remaining slots. -import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options" +import { CacheHint, type CacheExplicit, type CachePolicy } from "./schema/options" import { LLMRequest, Message, ToolDefinition, type ContentPart } from "./schema/messages" -const AUTO: CachePolicyObject = { +const AUTO: Omit = { tools: true, system: true, messages: { tail: 1 }, } -const NONE: CachePolicyObject = {} +const NONE: Omit = {} const BREAKPOINT_CAP = 4 // Resolution rules: @@ -29,8 +29,8 @@ const BREAKPOINT_CAP = 4 // - "auto" → tools + first/last system + final message boundary. // - "none" → no auto placement; manual `CacheHint`s still flow. // - object form → exactly what the caller asked for. -const resolve = (policy: CachePolicy | undefined): CachePolicyObject => { - if (policy === undefined || policy === "auto") return AUTO +const resolve = (policy: CachePolicy | undefined): Omit => { + if (policy === undefined || (policy !== "none" && policy.mode === "auto")) return AUTO if (policy === "none") return NONE return policy } @@ -103,7 +103,7 @@ const markMessageAt = ( const markMessages = ( messages: ReadonlyArray, - strategy: NonNullable, + strategy: NonNullable, hint: CacheHint, budget: Budget, ): ReadonlyArray => { @@ -133,7 +133,11 @@ const countHints = (request: LLMRequest) => export const applyCachePolicy = (request: LLMRequest): LLMRequest => { if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request - if (request.model.route.id === "openrouter" && (request.cache === undefined || request.cache === "auto")) return request + if ( + request.model.route.id === "openrouter" && + (request.cache === undefined || (request.cache !== "none" && request.cache.mode === "auto")) + ) + return request const policy = resolve(request.cache) if (!policy.tools && !policy.system && !policy.messages) return request diff --git a/packages/ai/src/protocols/open-responses.ts b/packages/ai/src/protocols/open-responses.ts index c9a86a915b..91a3dfc454 100644 --- a/packages/ai/src/protocols/open-responses.ts +++ b/packages/ai/src/protocols/open-responses.ts @@ -3,6 +3,7 @@ import type { Content } from "@opencode-ai/schema/tool" import { HttpTransport } from "../route/transport" import { Protocol } from "../route/protocol" import { + cacheKey, LLMError, LLMEvent, Usage, @@ -542,7 +543,7 @@ const lowerOptions = (request: LLMRequest) => { return { ...(options.instructions ? { instructions: options.instructions } : {}), ...(options.store !== undefined ? { store: options.store } : {}), - ...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}), + ...(cacheKey(request.cache) ? { prompt_cache_key: cacheKey(request.cache) } : {}), ...(options.include ? { include: options.include } : {}), ...(options.reasoningEffort || options.reasoningSummary ? { reasoning: { effort: options.reasoningEffort, summary: options.reasoningSummary } } diff --git a/packages/ai/src/providers/openrouter.ts b/packages/ai/src/providers/openrouter.ts index 935a6f05b4..4f454a917e 100644 --- a/packages/ai/src/providers/openrouter.ts +++ b/packages/ai/src/providers/openrouter.ts @@ -4,7 +4,7 @@ import { Endpoint } from "../route/endpoint" import { Framing } from "../route/framing" import { Protocol } from "../route/protocol" import { AuthOptions, type ProviderAuthOption } from "../route/auth-options" -import { ProviderID, type CacheHint, type ModelID, type ProviderOptions } from "../schema" +import { cacheKey, ProviderID, type CacheHint, type ModelID, type ProviderOptions } from "../schema" import type { ProviderPackage } from "../provider-package" import * as OpenAICompatibleProfiles from "./openai-compatible-profile" import * as OpenAIChat from "../protocols/openai-chat" @@ -121,7 +121,7 @@ export const protocol = Protocol.make({ ...body, messages, ...bodyOptions(request.providerOptions?.openrouter), - ...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}), + ...(cacheKey(request.cache) ? { prompt_cache_key: cacheKey(request.cache) } : {}), } as OpenRouterBody }), ), diff --git a/packages/ai/src/schema/messages.ts b/packages/ai/src/schema/messages.ts index bf8ffba3f0..d3689dc0e3 100644 --- a/packages/ai/src/schema/messages.ts +++ b/packages/ai/src/schema/messages.ts @@ -272,8 +272,6 @@ export class LLMRequest extends Schema.Class("LLM.Request")({ providerOptions: Schema.optional(ProviderOptions), http: Schema.optional(HttpOptions), cache: Schema.optional(CachePolicy), - // Stable cache affinity for protocols that support provider-managed prompt caching. - promptCacheKey: Schema.optional(Schema.String), metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), }) {} @@ -291,7 +289,6 @@ export namespace LLMRequest { providerOptions: request.providerOptions, http: request.http, cache: request.cache, - promptCacheKey: request.promptCacheKey, metadata: request.metadata, }) diff --git a/packages/ai/src/schema/options.ts b/packages/ai/src/schema/options.ts index 62c606056d..1ce02b35ac 100644 --- a/packages/ai/src/schema/options.ts +++ b/packages/ai/src/schema/options.ts @@ -256,18 +256,17 @@ export class CacheHint extends Schema.Class("LLM.CacheHint")({ ttlSeconds: Schema.optional(Schema.Number), }) {} -// Auto-placement policy for prompt caching. The protocol-neutral lowering step -// reads this and injects `CacheHint`s at the configured boundaries; the -// per-protocol body builders then translate those hints into wire markers as -// usual. `"auto"` is the recommended default for agent loops — it places -// breakpoints at the last tool definition, the first and last distinct system -// parts, and the conversation tail. The rolling message breakpoint keeps a -// prior cache entry within Anthropic/Bedrock's 20-block lookback during long -// tool loops. -// -// Pass `"none"` to opt out entirely (the legacy behavior). Pass the granular -// object form to override individual choices. -export const CachePolicyObject = Schema.Struct({ +const CacheKey = { key: Schema.optional(Schema.String) } + +export const CacheAuto = Schema.Struct({ + mode: Schema.Literal("auto"), + ...CacheKey, +}) +export type CacheAuto = Schema.Schema.Type + +export const CacheExplicit = Schema.Struct({ + mode: Schema.Literal("explicit"), + ...CacheKey, tools: Schema.optional(Schema.Boolean), system: Schema.optional(Schema.Boolean), messages: Schema.optional( @@ -279,7 +278,12 @@ export const CachePolicyObject = Schema.Struct({ ), ttlSeconds: Schema.optional(Schema.Number), }) -export type CachePolicyObject = Schema.Schema.Type +export type CacheExplicit = Schema.Schema.Type -export const CachePolicy = Schema.Union([Schema.Literal("auto"), Schema.Literal("none"), CachePolicyObject]) +// Omitted configuration uses automatic provider behavior and OpenCode's +// automatic breakpoint placement where required. `"none"` sends no cache key +// or explicit controls; providers may still cache implicitly. +export const CachePolicy = Schema.Union([Schema.Literal("none"), CacheAuto, CacheExplicit]) export type CachePolicy = Schema.Schema.Type + +export const cacheKey = (cache: CachePolicy | undefined) => (cache && cache !== "none" ? cache.key : undefined) diff --git a/packages/ai/test/cache-policy.test.ts b/packages/ai/test/cache-policy.test.ts index a862b8b65d..a5092c76d8 100644 --- a/packages/ai/test/cache-policy.test.ts +++ b/packages/ai/test/cache-policy.test.ts @@ -64,7 +64,7 @@ describe("applyCachePolicy", () => { Message.assistant("assistant reply"), Message.user("latest user message"), ], - cache: "auto", + cache: { mode: "auto" }, }), ) @@ -93,7 +93,7 @@ describe("applyCachePolicy", () => { model: openaiModel, system: "Sys", prompt: "hi", - cache: "auto", + cache: { mode: "auto" }, }), ) @@ -112,7 +112,7 @@ describe("applyCachePolicy", () => { model: geminiModel, system: "Sys", prompt: "hi", - cache: "auto", + cache: { mode: "auto" }, }), ) @@ -133,7 +133,7 @@ describe("applyCachePolicy", () => { ], tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }], messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")], - cache: "auto", + cache: { mode: "auto" }, }), ) @@ -183,7 +183,7 @@ describe("applyCachePolicy", () => { system: "Sys", tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }], prompt: "hi", - cache: { tools: true }, + cache: { mode: "explicit", tools: true }, }), ) @@ -204,7 +204,7 @@ describe("applyCachePolicy", () => { { type: "text", text: "last system" }, ], prompt: "hi", - cache: "auto", + cache: { mode: "auto" }, }), ) @@ -233,7 +233,7 @@ describe("applyCachePolicy", () => { ], tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }], prompt: "hi", - cache: "auto", + cache: { mode: "auto" }, }) const applied = applyCachePolicy(request) expect(applied.tools[0]?.cache).toBeDefined() @@ -267,7 +267,7 @@ describe("applyCachePolicy", () => { model: anthropicModel, system: "Sys", prompt: "hi", - cache: { system: true, ttlSeconds: 3600 }, + cache: { mode: "explicit", system: true, ttlSeconds: 3600 }, }), ) @@ -283,7 +283,7 @@ describe("applyCachePolicy", () => { LLM.request({ model: anthropicModel, messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2"), Message.assistant("a2")], - cache: { messages: { tail: 2 } }, + cache: { mode: "explicit", messages: { tail: 2 } }, }), ) @@ -301,7 +301,7 @@ describe("applyCachePolicy", () => { LLM.request({ model: anthropicModel, messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2")], - cache: { messages: "latest-assistant" }, + cache: { mode: "explicit", messages: "latest-assistant" }, }), ) diff --git a/packages/ai/test/provider-options/cloudflare.types.ts b/packages/ai/test/provider-options/cloudflare.types.ts index e98d5153ca..d0eef5b82e 100644 --- a/packages/ai/test/provider-options/cloudflare.types.ts +++ b/packages/ai/test/provider-options/cloudflare.types.ts @@ -3,11 +3,11 @@ import { CloudflareWorkersAI } from "../../src/providers" const model = CloudflareWorkersAI.configure({ accountId: "account", apiKey: "test" }).model("model") -LLM.request({ model, prompt: "Hello", promptCacheKey: "cache" }) +LLM.request({ model, prompt: "Hello", cache: { mode: "auto", key: "cache" } }) LLM.request({ model, prompt: "Hello", // @ts-expect-error Prompt cache keys must be strings. - promptCacheKey: 1, + cache: { mode: "auto", key: 1 }, }) diff --git a/packages/ai/test/provider/openai-responses-cache.recorded.test.ts b/packages/ai/test/provider/openai-responses-cache.recorded.test.ts index fa764f3535..5bf0b08e47 100644 --- a/packages/ai/test/provider/openai-responses-cache.recorded.test.ts +++ b/packages/ai/test/provider/openai-responses-cache.recorded.test.ts @@ -20,7 +20,7 @@ const cacheRequest = LLM.request({ system: LARGE_CACHEABLE_SYSTEM, prompt: "Say hi.", generation: { maxTokens: 16, temperature: 0 }, - promptCacheKey: "recorded-cache-test", + cache: { mode: "auto", key: "recorded-cache-test" }, }) const recorded = recordedTests({ diff --git a/packages/ai/test/provider/openai-responses.test.ts b/packages/ai/test/provider/openai-responses.test.ts index d42c972045..bc8229c208 100644 --- a/packages/ai/test/provider/openai-responses.test.ts +++ b/packages/ai/test/provider/openai-responses.test.ts @@ -680,7 +680,7 @@ describe("OpenAI Responses route", () => { LLM.request({ model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).model("gpt-5.2"), prompt: "think", - promptCacheKey: "session_123", + cache: { mode: "auto", key: "session_123" }, providerOptions: { openai: { reasoningEffort: "high", @@ -810,7 +810,7 @@ describe("OpenAI Responses route", () => { apiKey: "test", }).model("gpt-4.1-mini"), prompt: "no cache", - promptCacheKey: "request_cache", + cache: { mode: "auto", key: "request_cache" }, }), ) diff --git a/packages/ai/test/provider/openrouter.test.ts b/packages/ai/test/provider/openrouter.test.ts index 6077e5c797..a4a1101554 100644 --- a/packages/ai/test/provider/openrouter.test.ts +++ b/packages/ai/test/provider/openrouter.test.ts @@ -43,7 +43,7 @@ describe("OpenRouter", () => { ], tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }], prompt: "Hello", - cache: { tools: true, system: true, messages: { tail: 1 } }, + cache: { mode: "explicit", tools: true, system: true, messages: { tail: 1 } }, }), ) @@ -123,7 +123,7 @@ describe("OpenRouter", () => { const prepared = yield* compileRequest( LLM.request({ model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"), - cache: { messages: "latest-assistant" }, + cache: { mode: "explicit", messages: "latest-assistant" }, messages: [Message.user("Think"), Message.assistant([{ type: "reasoning", text: "Reasoning" }])], }), ) @@ -173,7 +173,7 @@ describe("OpenRouter", () => { }, }).model("anthropic/claude-3.7-sonnet:thinking"), prompt: "Think briefly.", - promptCacheKey: "session_123", + cache: { mode: "auto", key: "session_123" }, }), ) diff --git a/packages/core/src/session/compaction.ts b/packages/core/src/session/compaction.ts index 3c47a2b3e8..4770abcd32 100644 --- a/packages/core/src/session/compaction.ts +++ b/packages/core/src/session/compaction.ts @@ -258,7 +258,7 @@ const make = (dependencies: Dependencies) => { .stream( LLM.request({ model: plan.model, - promptCacheKey: SessionPromptCacheKey.make(plan.session.id), + cache: { mode: "auto", key: SessionPromptCacheKey.make(plan.session.id) }, http: { headers: SessionModelHeaders.make(plan.session, dependencies.app) }, messages: [Message.user(plan.prompt)], tools: [], diff --git a/packages/core/src/session/generate-node.ts b/packages/core/src/session/generate-node.ts index a36582ce38..7c092c985e 100644 --- a/packages/core/src/session/generate-node.ts +++ b/packages/core/src/session/generate-node.ts @@ -69,7 +69,7 @@ export const layer = Layer.effect( LLM.request({ model: model.model, http: { headers: SessionModelHeaders.make(selection.session, app) }, - promptCacheKey: SessionPromptCacheKey.make(selection.session.id), + cache: { mode: "auto", key: SessionPromptCacheKey.make(selection.session.id) }, system: contextEvent.system, messages: contextEvent.messages, tools: hookedTools, diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index 6e8c9306d5..c7f2f80b14 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -213,7 +213,7 @@ export const layer = Layer.effect( http: { headers: SessionModelHeaders.make(session, app), }, - promptCacheKey: SessionPromptCacheKey.make(session.id), + cache: { mode: "auto", key: SessionPromptCacheKey.make(session.id) }, system: contextEvent.system, messages: boundImages(unsupportedParts(contextEvent.messages, resolved.capabilities)), tools: hookedTools, diff --git a/packages/core/src/session/title.ts b/packages/core/src/session/title.ts index d4d134cf78..cde5ae3836 100644 --- a/packages/core/src/session/title.ts +++ b/packages/core/src/session/title.ts @@ -81,7 +81,7 @@ const make = (dependencies: Dependencies) => { .stream( LLM.request({ model: resolved.model, - promptCacheKey: SessionPromptCacheKey.make(session.id), + cache: { mode: "auto", key: SessionPromptCacheKey.make(session.id) }, http: { headers: SessionModelHeaders.make(session, dependencies.app) }, system: agent.system, messages: [Message.user(firstUser.text)], diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index 3dc66f549e..6baf9537f8 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -235,7 +235,7 @@ it.effect("manual compaction summarizes short context instead of no-op", () => expect(Array.from(yield* Fiber.join(delta)).map((event) => event.data.text)).toEqual(["manual summary"]) expect(requests).toHaveLength(1) - expect(requests[0]?.promptCacheKey).toBe(sessionID) + expect(requests[0]?.cache).toEqual({ mode: "auto", key: sessionID }) expect(requests[0]?.http?.headers).toEqual({ "x-session-affinity": sessionID, "X-Session-Id": sessionID, diff --git a/packages/core/test/session-generate.test.ts b/packages/core/test/session-generate.test.ts index f98438e5af..b898d87766 100644 --- a/packages/core/test/session-generate.test.ts +++ b/packages/core/test/session-generate.test.ts @@ -287,7 +287,7 @@ it.effect("generates from fresh settled Session context without durable mutation expect(requests[0]?.system[0]?.text).toBe("Hooked system") expect(requests[0]?.system.map((part) => part.text)).toContain("Initial context") expect(requests[0]?.http?.headers).toMatchObject({ "X-Session-Id": sessionID }) - expect(requests[0]?.promptCacheKey).toBe(sessionID) + expect(requests[0]?.cache).toEqual({ mode: "auto", key: sessionID }) const instructionUpdates = requests[0]?.messages.flatMap((message) => message.role === "system" ? message.content.flatMap((content) => (content.type === "text" ? [content.text] : [])) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 052048e7c9..d46e9c7601 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -3210,7 +3210,7 @@ describe("SessionRunnerLLM", () => { yield* stream.started expect(requests).toHaveLength(2) - expect(requests.map((request) => request.promptCacheKey)).toEqual([ + expect(requests.map((request) => (request.cache !== "none" ? request.cache?.key : undefined))).toEqual([ sessionID, otherSessionID, ]) @@ -3241,7 +3241,7 @@ describe("SessionRunnerLLM", () => { yield* session.resume(longSessionID) yield* session.resume(otherLongSessionID) - const keys = requests.map((request) => request.promptCacheKey) + const keys = requests.map((request) => (request.cache !== "none" ? request.cache?.key : undefined)) expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)]) expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true) expect(keys[0]).not.toBe(keys[1]) diff --git a/packages/core/test/session-title.test.ts b/packages/core/test/session-title.test.ts index d0cfc78b49..3d347b576b 100644 --- a/packages/core/test/session-title.test.ts +++ b/packages/core/test/session-title.test.ts @@ -153,7 +153,7 @@ it.effect("generates a title from the sole user message and renames the session" yield* title.generateForFirstPrompt(sessionID) expect(requests).toHaveLength(1) - expect(requests[0]?.promptCacheKey).toBe(sessionID) + expect(requests[0]?.cache).toEqual({ mode: "auto", key: sessionID }) expect(requests[0]?.http?.headers).toEqual({ "x-session-affinity": sessionID, "X-Session-Id": sessionID,