ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(ai): unify prompt cache configuration

Browse source
This commit is contained in:
Aiden Cline 2026-07-31 17:52:32 -05:00
commit a214ac39de
19 changed files with 64 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

9
packages/ai/example/tutorial.ts
View file

@ -33,9 +33,10 @@ const model = OpenAI.configure({
//
// - `generation`: common controls such as max tokens, temperature, topP/topK,
// penalties, seed, and stop sequences.
// - `cache`: provider-neutral prompt caching behavior and cache affinity.
// - `providerOptions`: namespaced provider-native behavior. For example,
// OpenAI cache keys and store behavior, Anthropic thinking, Gemini thinking
// config, or OpenRouter routing/reasoning.
// OpenAI store behavior, Anthropic thinking, Gemini thinking config, or
// OpenRouter routing/reasoning.
// - `http`: last-resort serializable overlays for final request body, headers,
// and query params. Prefer typed `providerOptions` when a field is stable.
//
@ -45,9 +46,7 @@ const request = LLM.request({
system: "You are concise and practical.",
prompt: "Tell me a joke",
generation: { maxTokens: 80, temperature: 0.7 },
providerOptions: {
openai: { promptCacheKey: "tutorial-joke" },
},
cache: { mode: "auto", key: "tutorial-joke" },
})
// 3. `generate` sends the request and collects the event stream into one

18
packages/ai/src/cache-policy.ts
View file

@ -10,16 +10,16 @@
//
// Manual `cache: CacheHint` placements on individual parts are preserved and
// count against the four-breakpoint budget; auto only fills remaining slots.
import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options"
import { CacheHint, type CacheExplicit, type CachePolicy } from "./schema/options"
import { LLMRequest, Message, ToolDefinition, type ContentPart } from "./schema/messages"
const AUTO: CachePolicyObject = {
const AUTO: Omit<CacheExplicit, "mode" | "key"> = {
tools: true,
system: true,
messages: { tail: 1 },
}
const NONE: CachePolicyObject = {}
const NONE: Omit<CacheExplicit, "mode" | "key"> = {}
const BREAKPOINT_CAP = 4
// Resolution rules:
@ -29,8 +29,8 @@ const BREAKPOINT_CAP = 4
// - "auto" → tools + first/last system + final message boundary.
// - "none" → no auto placement; manual `CacheHint`s still flow.
// - object form → exactly what the caller asked for.
const resolve = (policy: CachePolicy | undefined): CachePolicyObject => {
if (policy === undefined || policy === "auto") return AUTO
const resolve = (policy: CachePolicy | undefined): Omit<CacheExplicit, "mode" | "key"> => {
if (policy === undefined || (policy !== "none" && policy.mode === "auto")) return AUTO
if (policy === "none") return NONE
return policy
}
@ -103,7 +103,7 @@ const markMessageAt = (
const markMessages = (
messages: ReadonlyArray<Message>,
strategy: NonNullable<CachePolicyObject["messages"]>,
strategy: NonNullable<CacheExplicit["messages"]>,
hint: CacheHint,
budget: Budget,
): ReadonlyArray<Message> => {
@ -133,7 +133,11 @@ const countHints = (request: LLMRequest) =>
export const applyCachePolicy = (request: LLMRequest): LLMRequest => {
if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request
if (request.model.route.id === "openrouter" && (request.cache === undefined || request.cache === "auto")) return request
if (
request.model.route.id === "openrouter" &&
(request.cache === undefined || (request.cache !== "none" && request.cache.mode === "auto"))
)
return request
const policy = resolve(request.cache)
if (!policy.tools && !policy.system && !policy.messages) return request

3
packages/ai/src/protocols/open-responses.ts
View file

@ -3,6 +3,7 @@ import type { Content } from "@opencode-ai/schema/tool"
import { HttpTransport } from "../route/transport"
import { Protocol } from "../route/protocol"
import {
cacheKey,
LLMError,
LLMEvent,
Usage,
@ -542,7 +543,7 @@ const lowerOptions = (request: LLMRequest) => {
return {
...(options.instructions ? { instructions: options.instructions } : {}),
...(options.store !== undefined ? { store: options.store } : {}),
...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}),
...(cacheKey(request.cache) ? { prompt_cache_key: cacheKey(request.cache) } : {}),
...(options.include ? { include: options.include } : {}),
...(options.reasoningEffort || options.reasoningSummary
? { reasoning: { effort: options.reasoningEffort, summary: options.reasoningSummary } }

4
packages/ai/src/providers/openrouter.ts
View file

@ -4,7 +4,7 @@ import { Endpoint } from "../route/endpoint"
import { Framing } from "../route/framing"
import { Protocol } from "../route/protocol"
import { AuthOptions, type ProviderAuthOption } from "../route/auth-options"
import { ProviderID, type CacheHint, type ModelID, type ProviderOptions } from "../schema"
import { cacheKey, ProviderID, type CacheHint, type ModelID, type ProviderOptions } from "../schema"
import type { ProviderPackage } from "../provider-package"
import * as OpenAICompatibleProfiles from "./openai-compatible-profile"
import * as OpenAIChat from "../protocols/openai-chat"
@ -121,7 +121,7 @@ export const protocol = Protocol.make({
...body,
messages,
...bodyOptions(request.providerOptions?.openrouter),
...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}),
...(cacheKey(request.cache) ? { prompt_cache_key: cacheKey(request.cache) } : {}),
} as OpenRouterBody
}),
),

3
packages/ai/src/schema/messages.ts
View file

@ -272,8 +272,6 @@ export class LLMRequest extends Schema.Class<LLMRequest>("LLM.Request")({
providerOptions: Schema.optional(ProviderOptions),
http: Schema.optional(HttpOptions),
cache: Schema.optional(CachePolicy),
// Stable cache affinity for protocols that support provider-managed prompt caching.
promptCacheKey: Schema.optional(Schema.String),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}) {}
@ -291,7 +289,6 @@ export namespace LLMRequest {
providerOptions: request.providerOptions,
http: request.http,
cache: request.cache,
promptCacheKey: request.promptCacheKey,
metadata: request.metadata,
})

32
packages/ai/src/schema/options.ts
View file

@ -256,18 +256,17 @@ export class CacheHint extends Schema.Class<CacheHint>("LLM.CacheHint")({
ttlSeconds: Schema.optional(Schema.Number),
}) {}
// Auto-placement policy for prompt caching. The protocol-neutral lowering step
// reads this and injects `CacheHint`s at the configured boundaries; the
// per-protocol body builders then translate those hints into wire markers as
// usual. `"auto"` is the recommended default for agent loops — it places
// breakpoints at the last tool definition, the first and last distinct system
// parts, and the conversation tail. The rolling message breakpoint keeps a
// prior cache entry within Anthropic/Bedrock's 20-block lookback during long
// tool loops.
//
// Pass `"none"` to opt out entirely (the legacy behavior). Pass the granular
// object form to override individual choices.
export const CachePolicyObject = Schema.Struct({
const CacheKey = { key: Schema.optional(Schema.String) }
export const CacheAuto = Schema.Struct({
mode: Schema.Literal("auto"),
...CacheKey,
})
export type CacheAuto = Schema.Schema.Type<typeof CacheAuto>
export const CacheExplicit = Schema.Struct({
mode: Schema.Literal("explicit"),
...CacheKey,
tools: Schema.optional(Schema.Boolean),
system: Schema.optional(Schema.Boolean),
messages: Schema.optional(
@ -279,7 +278,12 @@ export const CachePolicyObject = Schema.Struct({
),
ttlSeconds: Schema.optional(Schema.Number),
})
export type CachePolicyObject = Schema.Schema.Type<typeof CachePolicyObject>
export type CacheExplicit = Schema.Schema.Type<typeof CacheExplicit>
export const CachePolicy = Schema.Union([Schema.Literal("auto"), Schema.Literal("none"), CachePolicyObject])
// Omitted configuration uses automatic provider behavior and OpenCode's
// automatic breakpoint placement where required. `"none"` sends no cache key
// or explicit controls; providers may still cache implicitly.
export const CachePolicy = Schema.Union([Schema.Literal("none"), CacheAuto, CacheExplicit])
export type CachePolicy = Schema.Schema.Type<typeof CachePolicy>
export const cacheKey = (cache: CachePolicy | undefined) => (cache && cache !== "none" ? cache.key : undefined)

20
packages/ai/test/cache-policy.test.ts
View file

@ -64,7 +64,7 @@ describe("applyCachePolicy", () => {
Message.assistant("assistant reply"),
Message.user("latest user message"),
],
cache: "auto",
cache: { mode: "auto" },
}),
)
@ -93,7 +93,7 @@ describe("applyCachePolicy", () => {
model: openaiModel,
system: "Sys",
prompt: "hi",
cache: "auto",
cache: { mode: "auto" },
}),
)
@ -112,7 +112,7 @@ describe("applyCachePolicy", () => {
model: geminiModel,
system: "Sys",
prompt: "hi",
cache: "auto",
cache: { mode: "auto" },
}),
)
@ -133,7 +133,7 @@ describe("applyCachePolicy", () => {
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")],
cache: "auto",
cache: { mode: "auto" },
}),
)
@ -183,7 +183,7 @@ describe("applyCachePolicy", () => {
system: "Sys",
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
cache: { tools: true },
cache: { mode: "explicit", tools: true },
}),
)
@ -204,7 +204,7 @@ describe("applyCachePolicy", () => {
{ type: "text", text: "last system" },
],
prompt: "hi",
cache: "auto",
cache: { mode: "auto" },
}),
)
@ -233,7 +233,7 @@ describe("applyCachePolicy", () => {
],
tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
prompt: "hi",
cache: "auto",
cache: { mode: "auto" },
})
const applied = applyCachePolicy(request)
expect(applied.tools[0]?.cache).toBeDefined()
@ -267,7 +267,7 @@ describe("applyCachePolicy", () => {
model: anthropicModel,
system: "Sys",
prompt: "hi",
cache: { system: true, ttlSeconds: 3600 },
cache: { mode: "explicit", system: true, ttlSeconds: 3600 },
}),
)
@ -283,7 +283,7 @@ describe("applyCachePolicy", () => {
LLM.request({
model: anthropicModel,
messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2"), Message.assistant("a2")],
cache: { messages: { tail: 2 } },
cache: { mode: "explicit", messages: { tail: 2 } },
}),
)
@ -301,7 +301,7 @@ describe("applyCachePolicy", () => {
LLM.request({
model: anthropicModel,
messages: [Message.user("u1"), Message.assistant("a1"), Message.user("u2")],
cache: { messages: "latest-assistant" },
cache: { mode: "explicit", messages: "latest-assistant" },
}),
)

4
packages/ai/test/provider-options/cloudflare.types.ts
View file

@ -3,11 +3,11 @@ import { CloudflareWorkersAI } from "../../src/providers"
const model = CloudflareWorkersAI.configure({ accountId: "account", apiKey: "test" }).model("model")
LLM.request({ model, prompt: "Hello", promptCacheKey: "cache" })
LLM.request({ model, prompt: "Hello", cache: { mode: "auto", key: "cache" } })
LLM.request({
model,
prompt: "Hello",
// @ts-expect-error Prompt cache keys must be strings.
promptCacheKey: 1,
cache: { mode: "auto", key: 1 },
})

2
packages/ai/test/provider/openai-responses-cache.recorded.test.ts
View file

@ -20,7 +20,7 @@ const cacheRequest = LLM.request({
system: LARGE_CACHEABLE_SYSTEM,
prompt: "Say hi.",
generation: { maxTokens: 16, temperature: 0 },
promptCacheKey: "recorded-cache-test",
cache: { mode: "auto", key: "recorded-cache-test" },
})
const recorded = recordedTests({

4
packages/ai/test/provider/openai-responses.test.ts
View file

@ -680,7 +680,7 @@ describe("OpenAI Responses route", () => {
LLM.request({
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).model("gpt-5.2"),
prompt: "think",
promptCacheKey: "session_123",
cache: { mode: "auto", key: "session_123" },
providerOptions: {
openai: {
reasoningEffort: "high",
@ -810,7 +810,7 @@ describe("OpenAI Responses route", () => {
apiKey: "test",
}).model("gpt-4.1-mini"),
prompt: "no cache",
promptCacheKey: "request_cache",
cache: { mode: "auto", key: "request_cache" },
}),
)

6
packages/ai/test/provider/openrouter.test.ts
View file

@ -43,7 +43,7 @@ describe("OpenRouter", () => {
],
tools: [{ name: "lookup", description: "Lookup", inputSchema: { type: "object", properties: {} } }],
prompt: "Hello",
cache: { tools: true, system: true, messages: { tail: 1 } },
cache: { mode: "explicit", tools: true, system: true, messages: { tail: 1 } },
}),
)
@ -123,7 +123,7 @@ describe("OpenRouter", () => {
const prepared = yield* compileRequest(
LLM.request({
model: OpenRouter.configure({ apiKey: "test-key" }).model("anthropic/claude-sonnet-4.6"),
cache: { messages: "latest-assistant" },
cache: { mode: "explicit", messages: "latest-assistant" },
messages: [Message.user("Think"), Message.assistant([{ type: "reasoning", text: "Reasoning" }])],
}),
)
@ -173,7 +173,7 @@ describe("OpenRouter", () => {
},
}).model("anthropic/claude-3.7-sonnet:thinking"),
prompt: "Think briefly.",
promptCacheKey: "session_123",
cache: { mode: "auto", key: "session_123" },
}),
)

2
packages/core/src/session/compaction.ts
View file

@ -258,7 +258,7 @@ const make = (dependencies: Dependencies) => {
.stream(
LLM.request({
model: plan.model,
promptCacheKey: SessionPromptCacheKey.make(plan.session.id),
cache: { mode: "auto", key: SessionPromptCacheKey.make(plan.session.id) },
http: { headers: SessionModelHeaders.make(plan.session, dependencies.app) },
messages: [Message.user(plan.prompt)],
tools: [],

2
packages/core/src/session/generate-node.ts
View file

@ -69,7 +69,7 @@ export const layer = Layer.effect(
LLM.request({
model: model.model,
http: { headers: SessionModelHeaders.make(selection.session, app) },
promptCacheKey: SessionPromptCacheKey.make(selection.session.id),
cache: { mode: "auto", key: SessionPromptCacheKey.make(selection.session.id) },
system: contextEvent.system,
messages: contextEvent.messages,
tools: hookedTools,

2
packages/core/src/session/model-request.ts
View file

@ -213,7 +213,7 @@ export const layer = Layer.effect(
http: {
headers: SessionModelHeaders.make(session, app),
},
promptCacheKey: SessionPromptCacheKey.make(session.id),
cache: { mode: "auto", key: SessionPromptCacheKey.make(session.id) },
system: contextEvent.system,
messages: boundImages(unsupportedParts(contextEvent.messages, resolved.capabilities)),
tools: hookedTools,

2
packages/core/src/session/title.ts
View file

@ -81,7 +81,7 @@ const make = (dependencies: Dependencies) => {
.stream(
LLM.request({
model: resolved.model,
promptCacheKey: SessionPromptCacheKey.make(session.id),
cache: { mode: "auto", key: SessionPromptCacheKey.make(session.id) },
http: { headers: SessionModelHeaders.make(session, dependencies.app) },
system: agent.system,
messages: [Message.user(firstUser.text)],

2
packages/core/test/session-compaction.test.ts
View file

@ -235,7 +235,7 @@ it.effect("manual compaction summarizes short context instead of no-op", () =>
expect(Array.from(yield* Fiber.join(delta)).map((event) => event.data.text)).toEqual(["manual summary"])
expect(requests).toHaveLength(1)
expect(requests[0]?.promptCacheKey).toBe(sessionID)
expect(requests[0]?.cache).toEqual({ mode: "auto", key: sessionID })
expect(requests[0]?.http?.headers).toEqual({
"x-session-affinity": sessionID,
"X-Session-Id": sessionID,

2
packages/core/test/session-generate.test.ts
View file

@ -287,7 +287,7 @@ it.effect("generates from fresh settled Session context without durable mutation
expect(requests[0]?.system[0]?.text).toBe("Hooked system")
expect(requests[0]?.system.map((part) => part.text)).toContain("Initial context")
expect(requests[0]?.http?.headers).toMatchObject({ "X-Session-Id": sessionID })
expect(requests[0]?.promptCacheKey).toBe(sessionID)
expect(requests[0]?.cache).toEqual({ mode: "auto", key: sessionID })
const instructionUpdates = requests[0]?.messages.flatMap((message) =>
message.role === "system"
? message.content.flatMap((content) => (content.type === "text" ? [content.text] : []))

4
packages/core/test/session-runner.test.ts
View file

@ -3210,7 +3210,7 @@ describe("SessionRunnerLLM", () => {
yield* stream.started
expect(requests).toHaveLength(2)
expect(requests.map((request) => request.promptCacheKey)).toEqual([
expect(requests.map((request) => (request.cache !== "none" ? request.cache?.key : undefined))).toEqual([
sessionID,
otherSessionID,
])
@ -3241,7 +3241,7 @@ describe("SessionRunnerLLM", () => {
yield* session.resume(longSessionID)
yield* session.resume(otherLongSessionID)
const keys = requests.map((request) => request.promptCacheKey)
const keys = requests.map((request) => (request.cache !== "none" ? request.cache?.key : undefined))
expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)])
expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true)
expect(keys[0]).not.toBe(keys[1])

2
packages/core/test/session-title.test.ts
View file

@ -153,7 +153,7 @@ it.effect("generates a title from the sole user message and renames the session"
yield* title.generateForFirstPrompt(sessionID)
expect(requests).toHaveLength(1)
expect(requests[0]?.promptCacheKey).toBe(sessionID)
expect(requests[0]?.cache).toEqual({ mode: "auto", key: sessionID })
expect(requests[0]?.http?.headers).toEqual({
"x-session-affinity": sessionID,
"X-Session-Id": sessionID,