diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index af9ac71a86..aa9d9e8590 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -2,7 +2,7 @@ export * as SessionModelRequest from "./model-request" import { LLM, Message, SystemPart, type LLMRequest, type ToolContent } from "@opencode-ai/ai" import { SessionError } from "@opencode-ai/schema/session-error" -import { Context, Effect, Layer } from "effect" +import { Context, Effect, Layer, LogLevel } from "effect" import { makeLocationNode } from "@opencode-ai/util/effect/app-node" import { App } from "../app" import { ModelV2 } from "../model" @@ -10,6 +10,7 @@ import { PluginHooks } from "../plugin/hooks" import { ToolRegistry } from "../tool/registry" import { SessionContext } from "./context" import { SessionModelHeaders } from "./model-headers" +import { PromptCacheDiagnostics } from "./prompt-cache-diagnostics" import { MAX_STEPS_PROMPT } from "./runner/max-steps" import PROMPT_DEFAULT from "./runner/prompt/base.txt" import { toLLMMessages } from "./runner/to-llm-message" @@ -87,6 +88,7 @@ export const layer = Layer.effect( Effect.gen(function* () { const hooks = yield* PluginHooks.Service const app = yield* App.Metadata + const promptCacheSnapshots = new Map() const prepare = Effect.fn("SessionModelRequest.prepare")(function* (input: PrepareInput) { const session = input.context.session @@ -134,6 +136,23 @@ export const layer = Layer.effect( tools: hookedTools, toolChoice: stepLimitReached ? "none" : undefined, }) + if (yield* LogLevel.isEnabled("Debug")) { + const current = PromptCacheDiagnostics.snapshot(request) + const comparison = PromptCacheDiagnostics.compare(promptCacheSnapshots.get(session.id), current) + promptCacheSnapshots.delete(session.id) + promptCacheSnapshots.set(session.id, current) + const oldest = promptCacheSnapshots.keys().next().value + if (promptCacheSnapshots.size > 100 && oldest !== undefined) promptCacheSnapshots.delete(oldest) + yield* Effect.logDebug("prompt cache prefix").pipe( + Effect.annotateLogs({ + sessionID: session.id, + toolCount: current.tools.length, + systemParts: current.system.length, + messageCount: current.messages.length, + ...comparison, + }), + ) + } const executeTool: ToolRegistry.ToolSet["execute"] = (executeInput) => { if (stepLimitReached) return Effect.succeed({ diff --git a/packages/core/src/session/prompt-cache-diagnostics.ts b/packages/core/src/session/prompt-cache-diagnostics.ts new file mode 100644 index 0000000000..81397668c3 --- /dev/null +++ b/packages/core/src/session/prompt-cache-diagnostics.ts @@ -0,0 +1,95 @@ +export * as PromptCacheDiagnostics from "./prompt-cache-diagnostics" + +import { createHash } from "node:crypto" +import type { LLMRequest } from "@opencode-ai/ai" + +interface Entry { + readonly label: string + readonly hash: string +} + +export interface Snapshot { + readonly settings: string + readonly tools: ReadonlyArray + readonly system: ReadonlyArray + readonly messages: ReadonlyArray +} + +export type Comparison = + | { readonly status: "initial" } + | { readonly status: "stable"; readonly messages: number } + | { readonly status: "append-only"; readonly previousMessages: number; readonly currentMessages: number } + | { + readonly status: "changed" + readonly component: "settings" | "tools" | "system" | "messages" + readonly index: number + readonly label: string + } + +const hash = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16) + +export function snapshot(request: LLMRequest): Snapshot { + return { + settings: hash({ + route: request.model.route.id, + provider: request.model.provider, + model: request.model.id, + modelDefaults: request.model.defaults, + compatibility: request.model.compatibility, + routeDefaults: { + generation: request.model.route.defaults.generation, + providerOptions: request.model.route.defaults.providerOptions, + http: request.model.route.defaults.http, + }, + generation: request.generation, + providerOptions: request.providerOptions, + http: request.http, + toolChoice: request.toolChoice, + cache: request.cache, + }), + tools: request.tools.map((tool) => ({ label: tool.name, hash: hash(tool) })), + system: request.system.map((part, index) => ({ label: `system[${index}]`, hash: hash(part) })), + messages: request.messages.map((message, index) => ({ + label: message.id ?? `${message.role}[${index}]`, + hash: hash(message), + })), + } +} + +export function compare(previous: Snapshot | undefined, current: Snapshot): Comparison { + if (!previous) return { status: "initial" } + if (previous.settings !== current.settings) + return { + status: "changed", + component: "settings", + index: 0, + label: "model settings", + } + const tools = firstChange(previous.tools, current.tools, false) + if (tools) return { status: "changed", component: "tools", ...tools } + const system = firstChange(previous.system, current.system, false) + if (system) return { status: "changed", component: "system", ...system } + const messages = firstChange(previous.messages, current.messages, true) + if (messages) return { status: "changed", component: "messages", ...messages } + if (previous.messages.length === current.messages.length) + return { status: "stable", messages: current.messages.length } + return { + status: "append-only", + previousMessages: previous.messages.length, + currentMessages: current.messages.length, + } +} + +function firstChange(previous: ReadonlyArray, current: ReadonlyArray, allowAppend: boolean) { + const index = previous.findIndex((entry, index) => entry.hash !== current[index]?.hash) + if (index >= 0) + return { + index, + label: current[index]?.label ?? previous[index]?.label ?? `entry[${index}]`, + } + if (current.length === previous.length || (allowAppend && current.length > previous.length)) return + return { + index: previous.length, + label: current[previous.length]?.label ?? `entry[${previous.length}]`, + } +} diff --git a/packages/core/test/prompt-cache-diagnostics.test.ts b/packages/core/test/prompt-cache-diagnostics.test.ts new file mode 100644 index 0000000000..104878ca7a --- /dev/null +++ b/packages/core/test/prompt-cache-diagnostics.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test" +import { GenerationOptions, LLM, LLMRequest, Message, Model, ToolDefinition } from "@opencode-ai/ai" +import { OpenAIChat } from "@opencode-ai/ai/protocols" +import { PromptCacheDiagnostics } from "@opencode-ai/core/session/prompt-cache-diagnostics" + +const model = Model.make({ id: "test", provider: "test", route: OpenAIChat.route }) +const tool = ToolDefinition.make({ + name: "read", + description: "Read a file", + inputSchema: { type: "object", properties: {} }, +}) + +const request = LLM.request({ + model, + system: "System", + prompt: "First", + tools: [tool], +}) + +describe("PromptCacheDiagnostics", () => { + test("distinguishes initial and stable requests", () => { + const snapshot = PromptCacheDiagnostics.snapshot(request) + expect(PromptCacheDiagnostics.compare(undefined, snapshot)).toEqual({ status: "initial" }) + expect(PromptCacheDiagnostics.compare(snapshot, snapshot)).toEqual({ status: "stable", messages: 1 }) + }) + + test("recognizes append-only history", () => { + const current = LLMRequest.update(request, { messages: [...request.messages, Message.assistant("Second")] }) + expect( + PromptCacheDiagnostics.compare( + PromptCacheDiagnostics.snapshot(request), + PromptCacheDiagnostics.snapshot(current), + ), + ).toEqual({ status: "append-only", previousMessages: 1, currentMessages: 2 }) + }) + + test("detects cache-sensitive setting changes", () => { + const current = LLMRequest.update(request, { generation: GenerationOptions.make({ temperature: 0.5 }) }) + expect( + PromptCacheDiagnostics.compare( + PromptCacheDiagnostics.snapshot(request), + PromptCacheDiagnostics.snapshot(current), + ), + ).toEqual({ status: "changed", component: "settings", index: 0, label: "model settings" }) + }) + + test("finds the first changed prefix component", () => { + const changedTool = ToolDefinition.make({ ...tool, description: "Read one file" }) + const current = LLMRequest.update(request, { tools: [changedTool] }) + expect( + PromptCacheDiagnostics.compare( + PromptCacheDiagnostics.snapshot(request), + PromptCacheDiagnostics.snapshot(current), + ), + ).toMatchObject({ status: "changed", component: "tools", index: 0, label: "read" }) + }) + + test("treats appended tools as a prefix change", () => { + const write = ToolDefinition.make({ + name: "write", + description: "Write a file", + inputSchema: { type: "object", properties: {} }, + }) + const current = LLMRequest.update(request, { tools: [...request.tools, write] }) + expect( + PromptCacheDiagnostics.compare( + PromptCacheDiagnostics.snapshot(request), + PromptCacheDiagnostics.snapshot(current), + ), + ).toMatchObject({ status: "changed", component: "tools", index: 1, label: "write" }) + }) +})