fix(ai): layer prompt cache breakpoints (#38725)

.changeset/quick-caches-roll.md

@@ -0,0 +1,5 @@
+---
+"@opencode-ai/ai": patch
+---
+
+Improve Anthropic and Bedrock prompt reuse with layered cache breakpoints that roll through long tool loops.

packages/ai/README.md

@@ -207,7 +207,9 @@ Prompt caching is **on by default**. Every `LLMRequest` resolves to `cache: "aut
 
 ### Auto placement
 
-`"auto"` places three breakpoints — last tool definition, last system part, latest user message. The last-user-message boundary is the load-bearing detail: in a tool-use loop, a single user turn expands into many assistant/tool round-trips, all sharing that prefix. Caching at that boundary lets every intra-turn API call hit.
+`"auto"` places up to four breakpoints — the last tool definition, the first system part, the last system part when distinct, and the final message boundary. These expose successively larger reusable prefixes for tools, the base agent, project instructions, and the active conversation. The rolling final-message boundary is the load-bearing detail in tool loops: it advances on every request so the previous cache entry stays within Anthropic's 20-block lookback.
+
+Tools precede every system and conversation block in the provider prefix, so tool definitions must remain byte-stable and deterministically ordered for downstream breakpoints to remain reusable.
 
 The math justifies the default: Anthropic's 5-minute cache write is 1.25× base, read is 0.1×, so a single reuse within 5 minutes already wins. One-shot completions below the per-model minimum-cacheable-token threshold silently no-op on the wire, so the worst case is harmless.
 
@@ -235,7 +237,7 @@ cache: {
 
 ### Manual hints
 
-Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints; it only fills gaps.
+Inline `CacheHint` on any text / system / tool / tool-result part overrides automatic placement. The auto policy preserves manual hints, counts them against Anthropic and Bedrock's four-breakpoint limit, and only fills the remaining slots.
 
 ```ts
 LLM.request({
@@ -251,8 +253,8 @@ LLM.request({
 
 | Protocol                | `cache: "auto"`                                                           |
 | ----------------------- | ------------------------------------------------------------------------- |
-| Anthropic Messages      | emits up to 3 `cache_control` markers (4-breakpoint cap enforced)         |
+| Anthropic Messages      | emits up to 4 `cache_control` markers (4-breakpoint cap enforced)         |
-| Bedrock Converse        | emits up to 3 `cachePoint` blocks (4-breakpoint cap enforced)             |
+| Bedrock Converse        | emits up to 4 `cachePoint` blocks (4-breakpoint cap enforced)             |
 | OpenAI Chat / Responses | no-op (implicit caching above 1024 tokens)                                |
 | Gemini                  | no-op (implicit caching on 2.5+; explicit `CachedContent` is out-of-band) |
 

packages/ai/src/cache-policy.ts

@@ -2,32 +2,31 @@
 // the policy designates. Runs once at compile time, before the per-protocol
 // body builder, so the existing inline-hint lowering path handles the rest.
 //
-// The default `"auto"` shape places one breakpoint at the last tool definition,
+// The default `"auto"` shape places breakpoints at the last tool definition,
-// one at the last system part, and one at the latest user message. This
+// the first and last distinct system parts, and the conversation tail. This
-// matches what production agent harnesses (LangChain's caching middleware,
+// exposes reusable tool, base-agent, project, and session prefixes while
-// kern-ai's 10x cost-reduction playbook) converge on for tool-use loops: the
+// advancing the tail after each tool result keeps the previous cache entry
-// latest user message stays put while a single turn explodes into many
+// within Anthropic's 20-block lookback during long agent turns.
-// assistant/tool round-trips, so caching at that boundary lets every
-// intra-turn API call hit the prefix.
 //
-// Manual `cache: CacheHint` placements on individual parts are preserved —
+// Manual `cache: CacheHint` placements on individual parts are preserved and
-// this function only fills gaps the caller left empty.
+// count against the four-breakpoint budget; auto only fills remaining slots.
 import { CacheHint, type CachePolicy, type CachePolicyObject } from "./schema/options"
 import { LLMRequest, Message, ToolDefinition, type ContentPart } from "./schema/messages"
 
 const AUTO: CachePolicyObject = {
   tools: true,
   system: true,
-  messages: "latest-user-message",
+  messages: { tail: 1 },
 }
 
 const NONE: CachePolicyObject = {}
+const BREAKPOINT_CAP = 4
 
 // Resolution rules:
 //   - undefined   → "auto" — caching is on by default. The math favors it:
 //                   Anthropic 5m-cache write is 1.25x base, read is 0.1x,
 //                   so a single reuse within 5 minutes already wins.
-//   - "auto"      → tools + system + latest user msg.
+//   - "auto"      → tools + first/last system + final message boundary.
 //   - "none"      → no auto placement; manual `CacheHint`s still flow.
 //   - object form → exactly what the caller asked for.
 const resolve = (policy: CachePolicy | undefined): CachePolicyObject => {
@@ -44,18 +43,32 @@ const RESPECTS_INLINE_HINTS = new Set(["anthropic-messages", "bedrock-converse"]
 const makeHint = (ttlSeconds: number | undefined): CacheHint =>
   ttlSeconds !== undefined ? new CacheHint({ type: "ephemeral", ttlSeconds }) : new CacheHint({ type: "ephemeral" })
 
-const markLastTool = (tools: ReadonlyArray<ToolDefinition>, hint: CacheHint): ReadonlyArray<ToolDefinition> => {
+interface Budget {
+  remaining: number
+}
+
+const markLastTool = (
+  tools: ReadonlyArray<ToolDefinition>,
+  hint: CacheHint,
+  budget: Budget,
+): ReadonlyArray<ToolDefinition> => {
   if (tools.length === 0) return tools
   const last = tools.length - 1
-  if (tools[last]!.cache) return tools
+  if (tools[last]!.cache || budget.remaining === 0) return tools
+  budget.remaining -= 1
   return tools.map((tool, i) => (i === last ? new ToolDefinition({ ...tool, cache: hint }) : tool))
 }
 
-const markLastSystem = (system: LLMRequest["system"], hint: CacheHint): LLMRequest["system"] => {
+const markSystemBoundaries = (system: LLMRequest["system"], hint: CacheHint, budget: Budget): LLMRequest["system"] => {
   if (system.length === 0) return system
-  const last = system.length - 1
+  let changed = false
-  if (system[last]!.cache) return system
+  const next = system.map((part, index) => {
-  return system.map((part, i) => (i === last ? { ...part, cache: hint } : part))
+    if ((index !== 0 && index !== system.length - 1) || part.cache || budget.remaining === 0) return part
+    budget.remaining -= 1
+    changed = true
+    return { ...part, cache: hint }
+  })
+  return changed ? next : system
 }
 
 const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]): number =>
@@ -64,14 +77,20 @@ const lastIndexOfRole = (messages: ReadonlyArray<Message>, role: Message["role"]
 // Mark the last text part of `messages[index]`. If no text part exists, mark
 // the last content part regardless of type — that's the breakpoint position
 // in tool-result-only messages too.
-const markMessageAt = (messages: ReadonlyArray<Message>, index: number, hint: CacheHint): ReadonlyArray<Message> => {
+const markMessageAt = (
+  messages: ReadonlyArray<Message>,
+  index: number,
+  hint: CacheHint,
+  budget: Budget,
+): ReadonlyArray<Message> => {
   if (index < 0 || index >= messages.length) return messages
   const target = messages[index]!
   if (target.content.length === 0) return messages
   const lastTextIndex = target.content.findLastIndex((part) => part.type === "text")
   const markAt = lastTextIndex >= 0 ? lastTextIndex : target.content.length - 1
   const existing = target.content[markAt]!
-  if ("cache" in existing && existing.cache) return messages
+  if (("cache" in existing && existing.cache) || budget.remaining === 0) return messages
+  budget.remaining -= 1
   const nextContent = target.content.map((part, i) => (i === markAt ? ({ ...part, cache: hint } as ContentPart) : part))
   const next = new Message({ ...target, content: nextContent })
   // Single pass over `messages`, substituting the one updated entry. Long
@@ -86,25 +105,42 @@ const markMessages = (
   messages: ReadonlyArray<Message>,
   strategy: NonNullable<CachePolicyObject["messages"]>,
   hint: CacheHint,
+  budget: Budget,
 ): ReadonlyArray<Message> => {
   if (messages.length === 0) return messages
-  if (strategy === "latest-user-message") return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint)
+  if (strategy === "latest-user-message")
-  if (strategy === "latest-assistant") return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint)
+    return markMessageAt(messages, lastIndexOfRole(messages, "user"), hint, budget)
+  if (strategy === "latest-assistant")
+    return markMessageAt(messages, lastIndexOfRole(messages, "assistant"), hint, budget)
   const start = Math.max(0, messages.length - strategy.tail)
   let next = messages
-  for (let i = start; i < messages.length; i++) next = markMessageAt(next, i, hint)
+  for (let i = start; i < messages.length; i++) next = markMessageAt(next, i, hint, budget)
   return next
 }
 
+const countHints = (request: LLMRequest) =>
+  request.tools.reduce((count, tool) => count + (tool.cache === undefined ? 0 : 1), 0) +
+  request.system.reduce((count, part) => count + (part.cache === undefined ? 0 : 1), 0) +
+  request.messages.reduce(
+    (count, message) =>
+      count +
+      message.content.reduce(
+        (contentCount, part) => contentCount + ("cache" in part && part.cache !== undefined ? 1 : 0),
+        0,
+      ),
+    0,
+  )
+
 export const applyCachePolicy = (request: LLMRequest): LLMRequest => {
   if (!RESPECTS_INLINE_HINTS.has(request.model.route.id)) return request
   const policy = resolve(request.cache)
   if (!policy.tools && !policy.system && !policy.messages) return request
 
   const hint = makeHint(policy.ttlSeconds)
-  const tools = policy.tools ? markLastTool(request.tools, hint) : request.tools
+  const budget = { remaining: Math.max(0, BREAKPOINT_CAP - countHints(request)) }
-  const system = policy.system ? markLastSystem(request.system, hint) : request.system
+  const tools = policy.tools ? markLastTool(request.tools, hint, budget) : request.tools
-  const messages = policy.messages ? markMessages(request.messages, policy.messages, hint) : request.messages
+  const system = policy.system ? markSystemBoundaries(request.system, hint, budget) : request.system
+  const messages = policy.messages ? markMessages(request.messages, policy.messages, hint, budget) : request.messages
 
   if (tools === request.tools && system === request.system && messages === request.messages) return request
   return LLMRequest.update(request, { tools, system, messages })

packages/ai/src/schema/options.ts

@@ -251,11 +251,11 @@ export class CacheHint extends Schema.Class<CacheHint>("LLM.CacheHint")({
 // Auto-placement policy for prompt caching. The protocol-neutral lowering step
 // reads this and injects `CacheHint`s at the configured boundaries; the
 // per-protocol body builders then translate those hints into wire markers as
-// usual. `"auto"` is the recommended default for agent loops — it places one
+// usual. `"auto"` is the recommended default for agent loops — it places
-// breakpoint at the last tool definition, one at the last system part, and one
+// breakpoints at the last tool definition, the first and last distinct system
-// at the latest user message. The combination of provider invalidation
+// parts, and the conversation tail. The rolling message breakpoint keeps a
-// hierarchy (tools → system → messages) and Anthropic/Bedrock's 20-block
+// prior cache entry within Anthropic/Bedrock's 20-block lookback during long
-// lookback means three trailing breakpoints reliably cover the static prefix.
+// tool loops.
 //
 // Pass `"none"` to opt out entirely (the legacy behavior). Pass the granular
 // object form to override individual choices.

packages/ai/test/cache-policy.test.ts

@@ -39,8 +39,8 @@ describe("applyCachePolicy", () => {
         }),
       )
 
-      // No explicit cache field → auto policy fires → last system part + latest
+      // A single system block is both the first and last boundary, so the auto
-      // user message both get cache_control markers.
+      // policy deduplicates it and still marks the conversation tail.
       expect(prepared.body).toMatchObject({
         system: [{ type: "text", text: "You are concise.", cache_control: { type: "ephemeral" } }],
         messages: [{ role: "user", content: [{ type: "text", text: "hi", cache_control: { type: "ephemeral" } }] }],
@@ -48,12 +48,15 @@ describe("applyCachePolicy", () => {
     }),
   )
 
-  it.effect("'auto' marks the last tool, last system part, and latest user message on Anthropic", () =>
+  it.effect("'auto' marks the last tool, first and last system parts, and final message boundary on Anthropic", () =>
     Effect.gen(function* () {
       const prepared = yield* LLMClient.prepare(
         LLM.request({
           model: anthropicModel,
-          system: "Sys A",
+          system: [
+            { type: "text", text: "Base agent" },
+            { type: "text", text: "Project instructions" },
+          ],
           tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
           messages: [
             Message.user("first user"),
@@ -66,7 +69,10 @@ describe("applyCachePolicy", () => {
 
       expect(prepared.body).toMatchObject({
         tools: [{ name: "t1", cache_control: { type: "ephemeral" } }],
-        system: [{ type: "text", text: "Sys A", cache_control: { type: "ephemeral" } }],
+        system: [
+          { type: "text", text: "Base agent", cache_control: { type: "ephemeral" } },
+          { type: "text", text: "Project instructions", cache_control: { type: "ephemeral" } },
+        ],
         messages: [
           { role: "user", content: [{ type: "text", text: "first user" }] },
           { role: "assistant", content: [{ type: "text", text: "assistant reply" }] },
@@ -120,7 +126,10 @@ describe("applyCachePolicy", () => {
       const prepared = yield* LLMClient.prepare(
         LLM.request({
           model: bedrockModel,
-          system: "Sys",
+          system: [
+            { type: "text", text: "Base agent" },
+            { type: "text", text: "Project instructions" },
+          ],
           tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
           messages: [Message.user("first user"), Message.assistant("reply"), Message.user("latest user")],
           cache: "auto",
@@ -131,7 +140,12 @@ describe("applyCachePolicy", () => {
         toolConfig: {
           tools: [{ toolSpec: { name: "t1" } }, { cachePoint: { type: "default" } }],
         },
-        system: [{ text: "Sys" }, { cachePoint: { type: "default" } }],
+        system: [
+          { text: "Base agent" },
+          { cachePoint: { type: "default" } },
+          { text: "Project instructions" },
+          { cachePoint: { type: "default" } },
+        ],
         messages: [
           { role: "user", content: [{ text: "first user" }] },
           { role: "assistant", content: [{ text: "reply" }] },
@@ -193,9 +207,55 @@ describe("applyCachePolicy", () => {
         }),
       )
 
-      const body = prepared.body as { system: Array<{ text: string; cache_control?: unknown }> }
+      const body = prepared.body as {
+        system: Array<{ text: string; cache_control?: unknown }>
+        messages: Array<{ content: Array<{ cache_control?: unknown }> }>
+      }
       expect(body.system[0]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
       expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral" })
+      expect(body.messages[0]?.content[0]?.cache_control).toEqual({ type: "ephemeral" })
+    }),
+  )
+
+  it.effect("auto policy stays within the four-breakpoint cap when preserving manual hints", () =>
+    Effect.gen(function* () {
+      const request = LLM.request({
+        model: anthropicModel,
+        system: [
+          { type: "text", text: "Base agent" },
+          {
+            type: "text",
+            text: "Manual context",
+            cache: new CacheHint({ type: "ephemeral", ttlSeconds: 3600 }),
+          },
+          { type: "text", text: "Project instructions" },
+        ],
+        tools: [{ name: "t1", description: "t1", inputSchema: { type: "object", properties: {} } }],
+        prompt: "hi",
+        cache: "auto",
+      })
+      const applied = applyCachePolicy(request)
+      expect(applied.tools[0]?.cache).toBeDefined()
+      expect(applied.system.map((part) => part.cache !== undefined)).toEqual([true, true, true])
+      const tail = applied.messages[0]!.content[0]!
+      expect("cache" in tail ? tail.cache : undefined).toBeUndefined()
+      expect(applyCachePolicy(applied)).toBe(applied)
+
+      const prepared = yield* LLMClient.prepare(request)
+
+      const body = prepared.body as {
+        tools: Array<{ cache_control?: unknown }>
+        system: Array<{ cache_control?: unknown }>
+        messages: Array<{ content: Array<{ cache_control?: unknown }> }>
+      }
+      const marked = [
+        ...body.tools.map((tool) => tool.cache_control),
+        ...body.system.map((part) => part.cache_control),
+        ...body.messages.flatMap((message) => message.content.map((part) => part.cache_control)),
+      ].filter((cache) => cache !== undefined)
+      expect(marked).toHaveLength(4)
+      expect(body.system[1]?.cache_control).toEqual({ type: "ephemeral", ttl: "1h" })
+      expect(body.messages[0]?.content[0]?.cache_control).toBeUndefined()
     }),
   )
 

packages/ai/test/provider/anthropic-messages-cache.recorded.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect } from "bun:test"
 import { Effect } from "effect"
-import { CacheHint, LLM } from "../../src"
+import { CacheHint, LLM, LLMRequest, Message, ToolCallPart, ToolDefinition } from "../../src"
 import { LLMClient } from "../../src/route"
 import * as Anthropic from "../../src/providers/anthropic"
 import { LARGE_CACHEABLE_SYSTEM } from "../recorded-scenarios"
@@ -24,6 +24,39 @@ const cacheRequest = LLM.request({
   generation: { maxTokens: 16, temperature: 0 },
 })
 
+const lookup = ToolDefinition.make({
+  name: "lookup",
+  description: "Look up a fixture value.",
+  inputSchema: {
+    type: "object",
+    properties: { index: { type: "number" } },
+    required: ["index"],
+    additionalProperties: false,
+  },
+})
+const longToolTurn = [
+  Message.user("Run the fixture lookups."),
+  ...Array.from({ length: 11 }, (_, index) => {
+    const id = `lookup_${index}`
+    return [
+      Message.assistant(ToolCallPart.make({ id, name: lookup.name, input: { index } })),
+      Message.tool({
+        id,
+        name: lookup.name,
+        result: `Fixture result ${index}. `.repeat(80),
+      }),
+    ]
+  }).flat(),
+]
+const longToolTurnRequest = LLM.request({
+  id: "recorded_anthropic_cache_long_tool_turn",
+  model,
+  system: LARGE_CACHEABLE_SYSTEM,
+  messages: longToolTurn,
+  tools: [lookup],
+  generation: { maxTokens: 16, temperature: 0 },
+})
+
 const recorded = recordedTests({
   prefix: "anthropic-messages-cache",
   provider: "anthropic",
@@ -50,4 +83,28 @@ describe("Anthropic Messages cache recorded", () => {
       expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThan(0)
     }),
   )
+
+  recorded.effect.with("keeps a long tool turn inside the cache lookback", { tags: ["cache", "tool"] }, () =>
+    Effect.gen(function* () {
+      const first = yield* LLMClient.generate(longToolTurnRequest)
+      const firstRead = first.usage?.cacheReadInputTokens ?? 0
+      const firstWrite = first.usage?.cacheWriteInputTokens ?? 0
+      const firstCached = firstRead + firstWrite
+      // The prefix may already be warm when recording, so either a read or a
+      // write establishes that Anthropic recognized the cache boundary.
+      expect(firstCached).toBeGreaterThan(0)
+
+      const second = yield* LLMClient.generate(
+        LLMRequest.update(longToolTurnRequest, {
+          messages: [
+            ...longToolTurn,
+            Message.assistant("The fixture lookups are complete."),
+            Message.user("Reply exactly: OK"),
+          ],
+        }),
+      )
+      expect(second.usage?.cacheReadInputTokens ?? 0).toBeGreaterThanOrEqual(firstCached)
+      expect(second.usage?.cacheWriteInputTokens ?? 0).toBeLessThan(firstCached)
+    }),
+  )
 })

packages/ai/test/provider/bedrock-converse.test.ts

@@ -939,6 +939,7 @@ describe("Bedrock Converse route", () => {
       const prepared = yield* LLMClient.prepare<BedrockConverse.BedrockConverseBody>(
         LLM.request({
           model,
+          cache: "none",
           messages: [
             Message.assistant([ToolCallPart.make({ id: "call_1", name: "read", input: { path: "report.pdf" } })]),
             Message.tool({