ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(ai): promote prompt cache key

Browse source
This commit is contained in:
Aiden Cline 2026-07-31 16:59:24 -05:00
commit f6ea6b1762
21 changed files with 33 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/ai/src/protocols/open-responses.ts
View file

@ -542,7 +542,7 @@ const lowerOptions = (request: LLMRequest) => {
return {
...(options.instructions ? { instructions: options.instructions } : {}),
...(options.store !== undefined ? { store: options.store } : {}),
...(options.promptCacheKey ? { prompt_cache_key: options.promptCacheKey } : {}),
...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}),
...(options.include ? { include: options.include } : {}),
...(options.reasoningEffort || options.reasoningSummary
? { reasoning: { effort: options.reasoningEffort, summary: options.reasoningSummary } }

2
packages/ai/src/protocols/utils/open-responses-options.ts
View file

@ -33,7 +33,6 @@ export const ServiceTierSchema = Schema.Literals(ServiceTiers)
export interface Resolved {
readonly instructions?: string
readonly store?: boolean
readonly promptCacheKey?: string
readonly reasoningEffort?: string
readonly reasoningSummary?: "auto" | "concise" | "detailed"
readonly include?: ReadonlyArray<ResponseIncludable>
@ -50,7 +49,6 @@ export const resolve = (request: LLMRequest): Resolved => {
return {
instructions: typeof input?.instructions === "string" ? input.instructions : undefined,
store: typeof input?.store === "boolean" ? input.store : undefined,
promptCacheKey: typeof input?.promptCacheKey === "string" ? input.promptCacheKey : undefined,
reasoningEffort: typeof input?.reasoningEffort === "string" ? input.reasoningEffort : undefined,
reasoningSummary:
reasoningSummary === "auto" || reasoningSummary === "concise" || reasoningSummary === "detailed"

1
packages/ai/src/providers/open-responses-options.ts
View file

@ -5,7 +5,6 @@ export interface OpenResponsesOptionsInput {
readonly [key: string]: unknown
readonly instructions?: string
readonly store?: boolean
readonly promptCacheKey?: string
readonly reasoningEffort?: ReasoningEffort
readonly reasoningSummary?: "auto" | "concise" | "detailed"
readonly include?: ReadonlyArray<ResponseIncludable>

1
packages/ai/src/providers/openai-options.ts
View file

@ -17,7 +17,6 @@ const openAIProviderOptions = (options: OpenAIOptionsInput | undefined): Provide
const openai = Object.fromEntries(
definedEntries({
store: options?.store,
promptCacheKey: options?.promptCacheKey,
reasoningEffort: options?.reasoningEffort,
reasoningSummary: options?.reasoningSummary,
include: options?.include,

3
packages/ai/src/providers/openrouter.ts
View file

@ -55,7 +55,6 @@ export interface OpenRouterOptions {
readonly debug?: Readonly<{ echo_upstream_body?: boolean }>
readonly models?: ReadonlyArray<string>
readonly plugins?: ReadonlyArray<OpenRouterPlugin>
readonly promptCacheKey?: string
readonly provider?: OpenRouterProviderRouting
readonly reasoning?: Readonly<{
enabled?: boolean
@ -122,6 +121,7 @@ export const protocol = Protocol.make({
...body,
messages,
...bodyOptions(request.providerOptions?.openrouter),
...(request.promptCacheKey ? { prompt_cache_key: request.promptCacheKey } : {}),
} as OpenRouterBody
}),
),
@ -161,7 +161,6 @@ const bodyOptions = (input: unknown) => {
...(isRecord(debug) ? { debug } : {}),
...(typeof user === "string" ? { user } : {}),
...(isRecord(reasoning) ? { reasoning } : {}),
...(typeof promptCacheKey === "string" ? { prompt_cache_key: promptCacheKey } : {}),
}
}

3
packages/ai/src/schema/messages.ts
View file

@ -272,6 +272,8 @@ export class LLMRequest extends Schema.Class<LLMRequest>("LLM.Request")({
providerOptions: Schema.optional(ProviderOptions),
http: Schema.optional(HttpOptions),
cache: Schema.optional(CachePolicy),
// Stable cache affinity for protocols that support provider-managed prompt caching.
promptCacheKey: Schema.optional(Schema.String),
metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)),
}) {}
@ -289,6 +291,7 @@ export namespace LLMRequest {
providerOptions: request.providerOptions,
http: request.http,
cache: request.cache,
promptCacheKey: request.promptCacheKey,
metadata: request.metadata,
})

6
packages/ai/test/provider-options/cloudflare.types.ts
View file

@ -3,11 +3,11 @@ import { CloudflareWorkersAI } from "../../src/providers"
const model = CloudflareWorkersAI.configure({ accountId: "account", apiKey: "test" }).model("model")
LLM.request({ model, prompt: "Hello", providerOptions: { openai: { promptCacheKey: "cache" } } })
LLM.request({ model, prompt: "Hello", promptCacheKey: "cache" })
LLM.request({
model,
prompt: "Hello",
// @ts-expect-error Cloudflare's OpenAI-compatible prompt cache key must be a string.
providerOptions: { openai: { promptCacheKey: 1 } },
// @ts-expect-error Prompt cache keys must be strings.
promptCacheKey: 1,
})

2
packages/ai/test/provider/openai-responses-cache.recorded.test.ts
View file

@ -20,7 +20,7 @@ const cacheRequest = LLM.request({
system: LARGE_CACHEABLE_SYSTEM,
prompt: "Say hi.",
generation: { maxTokens: 16, temperature: 0 },
providerOptions: { openai: { promptCacheKey: "recorded-cache-test" } },
promptCacheKey: "recorded-cache-test",
})
const recorded = recordedTests({

7
packages/ai/test/provider/openai-responses.test.ts
View file

@ -680,9 +680,9 @@ describe("OpenAI Responses route", () => {
LLM.request({
model: OpenAI.configure({ baseURL: "https://api.openai.test/v1/", apiKey: "test" }).model("gpt-5.2"),
prompt: "think",
promptCacheKey: "session_123",
providerOptions: {
openai: {
promptCacheKey: "session_123",
reasoningEffort: "high",
reasoningSummary: "auto",
include: ["reasoning.encrypted_content"],
@ -801,17 +801,16 @@ describe("OpenAI Responses route", () => {
}),
)
it.effect("request OpenAI provider options override route defaults", () =>
it.effect("maps the request prompt cache key", () =>
Effect.gen(function* () {
const prepared = yield* compileRequest(
LLM.request({
model: OpenAI.configure({
baseURL: "https://api.openai.test/v1/",
apiKey: "test",
providerOptions: { openai: { promptCacheKey: "model_cache" } },
}).model("gpt-4.1-mini"),
prompt: "no cache",
providerOptions: { openai: { promptCacheKey: "request_cache" } },
promptCacheKey: "request_cache",
}),
)

2
packages/ai/test/provider/openrouter.test.ts
View file

@ -162,7 +162,6 @@ describe("OpenRouter", () => {
openrouter: {
usage: true,
reasoning: { effort: "high" },
promptCacheKey: "session_123",
models: ["anthropic/claude-sonnet-4.6", "google/gemini-3.1-pro"],
provider: { order: ["anthropic", "google"], require_parameters: true },
plugins: [{ id: "response-healing" }],
@ -174,6 +173,7 @@ describe("OpenRouter", () => {
},
}).model("anthropic/claude-3.7-sonnet:thinking"),
prompt: "Think briefly.",
promptCacheKey: "session_123",
}),
)

2
packages/core/src/aisdk-native.ts
View file

@ -108,6 +108,7 @@ function mapOpenRouterOptions(settings: Readonly<Record<string, unknown>>) {
"extraBody",
"fetch",
"headers",
"promptCacheKey",
"timeout",
].includes(key),
),
@ -124,7 +125,6 @@ function mapXAIOptions(settings: Readonly<Record<string, unknown>>) {
const options = {
...(typeof settings.reasoningEffort === "string" ? { reasoningEffort: settings.reasoningEffort } : {}),
...(typeof settings.store === "boolean" ? { store: settings.store } : {}),
...(typeof settings.promptCacheKey === "string" ? { promptCacheKey: settings.promptCacheKey } : {}),
}
if (Object.keys(options).length === 0) return {}
return { providerOptions: { xai: options } }

2
packages/core/src/session/compaction.ts
View file

@ -10,6 +10,7 @@ import { llmClient } from "../effect/app-node-platform"
import { SessionEvent } from "./event"
import type { SessionMessage } from "./message"
import { SessionModelHeaders } from "./model-headers"
import { SessionPromptCacheKey } from "./prompt-cache-key"
import { App } from "../app"
import { SessionRunnerModel } from "./runner/model"
import { SessionSchema } from "./schema"
@ -257,6 +258,7 @@ const make = (dependencies: Dependencies) => {
.stream(
LLM.request({
model: plan.model,
promptCacheKey: SessionPromptCacheKey.make(plan.session.id),
http: { headers: SessionModelHeaders.make(plan.session, dependencies.app) },
messages: [Message.user(plan.prompt)],
tools: [],

6
packages/core/src/session/generate-node.ts
View file

@ -11,6 +11,7 @@ import { SessionContext } from "./context"
import { SessionGenerate } from "./generate"
import { SessionHistory } from "./history"
import { SessionModelHeaders } from "./model-headers"
import { SessionPromptCacheKey } from "./prompt-cache-key"
import { SessionRunnerModel } from "./runner/model"
import PROMPT_DEFAULT from "./runner/prompt/base.txt"
import { toLLMMessages } from "./runner/to-llm-message"
@ -31,9 +32,6 @@ export const layer = Layer.effect(
const model = yield* models.resolve(selection.session)
const history = yield* SessionHistory.preview(database.db, selection.session.id, selection.instructions)
const providerMetadataKey = model.model.route.providerMetadataKey ?? model.model.provider
const promptCacheKey = /^ses_[0-9a-f]{64}$/.test(selection.session.id)
? selection.session.id.slice(4)
: selection.session.id
const tools = selection.tools
const toolDefinitions = tools.definitions
const toolsByName = new Map(toolDefinitions.map((tool) => [tool.name, tool]))
@ -71,7 +69,7 @@ export const layer = Layer.effect(
LLM.request({
model: model.model,
http: { headers: SessionModelHeaders.make(selection.session, app) },
providerOptions: { [providerMetadataKey]: { promptCacheKey } },
promptCacheKey: SessionPromptCacheKey.make(selection.session.id),
system: contextEvent.system,
messages: contextEvent.messages,
tools: hookedTools,

4
packages/core/src/session/model-request.ts
View file

@ -14,6 +14,7 @@ import { QuestionTool } from "../tool/plugin/question"
import { Tool } from "../tool"
import { SessionContext } from "./context"
import { SessionModelHeaders } from "./model-headers"
import { SessionPromptCacheKey } from "./prompt-cache-key"
import { PromptCacheDiagnostics } from "./prompt-cache-diagnostics"
import { MAX_STEPS_PROMPT } from "./runner/max-steps"
import PROMPT_DEFAULT from "./runner/prompt/base.txt"
@ -183,7 +184,6 @@ export const layer = Layer.effect(
// The final Step keeps definitions available to protocols with native "none",
// preserving their prompt cache prefix. Calls are still rejected at execution.
const tools = input.context.tools
const promptCacheKey = /^ses_[0-9a-f]{64}$/.test(session.id) ? session.id.slice(4) : session.id
const system = [agent.info.system ? agent.info.system : PROMPT_DEFAULT, input.context.initial]
.filter((part) => part.length > 0)
.map(SystemPart.make)
@ -213,7 +213,7 @@ export const layer = Layer.effect(
http: {
headers: SessionModelHeaders.make(session, app),
},
providerOptions: { [providerMetadataKey]: { promptCacheKey } },
promptCacheKey: SessionPromptCacheKey.make(session.id),
system: contextEvent.system,
messages: boundImages(unsupportedParts(contextEvent.messages, resolved.capabilities)),
tools: hookedTools,

6
packages/core/src/session/prompt-cache-key.ts Normal file
View file

@ -0,0 +1,6 @@
export * as SessionPromptCacheKey from "./prompt-cache-key"
import { SessionSchema } from "./schema"
export const make = (sessionID: SessionSchema.ID) =>
/^ses_[0-9a-f]{64}$/.test(sessionID) ? sessionID.slice(4) : sessionID

2
packages/core/src/session/title.ts
View file

@ -12,6 +12,7 @@ import { llmClient } from "../effect/app-node-platform"
import { SessionEvent } from "./event"
import { SessionHistory } from "./history"
import { SessionModelHeaders } from "./model-headers"
import { SessionPromptCacheKey } from "./prompt-cache-key"
import { SessionRunnerModel } from "./runner/model"
import { SessionSchema } from "./schema"
import { SessionUsage } from "./usage"
@ -80,6 +81,7 @@ const make = (dependencies: Dependencies) => {
.stream(
LLM.request({
model: resolved.model,
promptCacheKey: SessionPromptCacheKey.make(session.id),
http: { headers: SessionModelHeaders.make(session, dependencies.app) },
system: agent.system,
messages: [Message.user(firstUser.text)],

4
packages/core/test/aisdk-native.test.ts
View file

@ -13,7 +13,6 @@ describe("AISDKNative", () => {
models: ["anthropic/claude-sonnet-4.6"],
provider: { only: ["anthropic"], require_parameters: true },
reasoning: { effort: "high" },
promptCacheKey: "session_123",
future_option: { enabled: true },
}),
).toEqual({
@ -24,7 +23,6 @@ describe("AISDKNative", () => {
models: ["anthropic/claude-sonnet-4.6"],
provider: { only: ["anthropic"], require_parameters: true },
reasoning: { effort: "high" },
promptCacheKey: "session_123",
future_option: { enabled: true },
},
},
@ -105,7 +103,6 @@ describe("AISDKNative", () => {
baseURL: "https://xai.example/v1",
reasoningEffort: "custom",
store: true,
promptCacheKey: "cache-key",
}),
).toEqual({
package: "@opencode-ai/ai/providers/xai",
@ -116,7 +113,6 @@ describe("AISDKNative", () => {
xai: {
reasoningEffort: "custom",
store: true,
promptCacheKey: "cache-key",
},
},
},

1
packages/core/test/session-compaction.test.ts
View file

@ -235,6 +235,7 @@ it.effect("manual compaction summarizes short context instead of no-op", () =>
expect(Array.from(yield* Fiber.join(delta)).map((event) => event.data.text)).toEqual(["manual summary"])
expect(requests).toHaveLength(1)
expect(requests[0]?.promptCacheKey).toBe(sessionID)
expect(requests[0]?.http?.headers).toEqual({
"x-session-affinity": sessionID,
"X-Session-Id": sessionID,

2
packages/core/test/session-generate.test.ts
View file

@ -287,7 +287,7 @@ it.effect("generates from fresh settled Session context without durable mutation
expect(requests[0]?.system[0]?.text).toBe("Hooked system")
expect(requests[0]?.system.map((part) => part.text)).toContain("Initial context")
expect(requests[0]?.http?.headers).toMatchObject({ "X-Session-Id": sessionID })
expect(requests[0]?.providerOptions).toMatchObject({ openai: { promptCacheKey: sessionID } })
expect(requests[0]?.promptCacheKey).toBe(sessionID)
const instructionUpdates = requests[0]?.messages.flatMap((message) =>
message.role === "system"
? message.content.flatMap((content) => (content.type === "text" ? [content.text] : []))

4
packages/core/test/session-runner.test.ts
View file

@ -3210,7 +3210,7 @@ describe("SessionRunnerLLM", () => {
yield* stream.started
expect(requests).toHaveLength(2)
expect(requests.map((request) => request.providerOptions?.openai?.promptCacheKey)).toEqual([
expect(requests.map((request) => request.promptCacheKey)).toEqual([
sessionID,
otherSessionID,
])
@ -3241,7 +3241,7 @@ describe("SessionRunnerLLM", () => {
yield* session.resume(longSessionID)
yield* session.resume(otherLongSessionID)
const keys = requests.map((request) => request.providerOptions?.openai?.promptCacheKey)
const keys = requests.map((request) => request.promptCacheKey)
expect(keys).toEqual([longSessionID.slice(4), otherLongSessionID.slice(4)])
expect(keys.every((key) => typeof key === "string" && key.length === 64)).toBe(true)
expect(keys[0]).not.toBe(keys[1])

1
packages/core/test/session-title.test.ts
View file

@ -153,6 +153,7 @@ it.effect("generates a title from the sole user message and renames the session"
yield* title.generateForFirstPrompt(sessionID)
expect(requests).toHaveLength(1)
expect(requests[0]?.promptCacheKey).toBe(sessionID)
expect(requests[0]?.http?.headers).toEqual({
"x-session-affinity": sessionID,
"X-Session-Id": sessionID,