From caf727ecb7d255e741f72405e20d914ccb1c17c2 Mon Sep 17 00:00:00 2001 From: "opencode-agent[bot]" <219766164+opencode-agent[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:44:10 -0500 Subject: [PATCH] fix(core): filter unsupported media inputs (#38145) Co-authored-by: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> --- packages/core/src/session/model-request.ts | 44 ++++++++++++- packages/core/src/session/runner/model.ts | 17 ++++- packages/core/test/session-compact.test.ts | 9 ++- packages/core/test/session-compaction.test.ts | 8 ++- packages/core/test/session-generate.test.ts | 9 ++- .../core/test/session-model-request.test.ts | 64 +++++++++++++++++++ .../core/test/session-runner-recorded.test.ts | 9 ++- packages/core/test/session-runner.test.ts | 9 +-- packages/core/test/session-title.test.ts | 8 ++- 9 files changed, 163 insertions(+), 14 deletions(-) create mode 100644 packages/core/test/session-model-request.test.ts diff --git a/packages/core/src/session/model-request.ts b/packages/core/src/session/model-request.ts index 73fef668e0..7e5f84d20c 100644 --- a/packages/core/src/session/model-request.ts +++ b/packages/core/src/session/model-request.ts @@ -1,10 +1,11 @@ export * as SessionModelRequest from "./model-request" -import { LLM, Message, SystemPart, type LLMRequest } from "@opencode-ai/ai" +import { LLM, Message, SystemPart, type LLMRequest, type ToolContent } from "@opencode-ai/ai" import { SessionError } from "@opencode-ai/schema/session-error" import { Context, Effect, Layer } from "effect" import { makeLocationNode } from "@opencode-ai/util/effect/app-node" import { Client } from "@opencode-ai/util/client" +import { ModelV2 } from "../model" import { PluginHooks } from "../plugin/hooks" import { ToolRegistry } from "../tool/registry" import { SessionContext } from "./context" @@ -27,6 +28,45 @@ interface PrepareInput { readonly step: number } +const mimeToModality = (mime: string) => { + if (mime.startsWith("image/")) return "image" + if (mime.startsWith("audio/")) return "audio" + if (mime.startsWith("video/")) return "video" + if (mime === "application/pdf") return "pdf" +} + +const unsupportedMedia = (mime: string, name: string | undefined, capabilities: ModelV2.Capabilities) => { + const modality = mimeToModality(mime) + if (!modality || capabilities.input.some((item) => item.startsWith(modality))) return + return { + type: "text" as const, + text: `ERROR: Cannot read ${name ? `"${name}"` : modality} (this model does not support ${modality} input). Inform the user.`, + } +} + +export const unsupportedParts = (messages: LLMRequest["messages"], capabilities: ModelV2.Capabilities) => + messages.map((message) => + Message.make({ + ...message, + content: message.content.map((part) => { + if (part.type === "media") { + return unsupportedMedia(part.mediaType, part.filename, capabilities) ?? part + } + if (part.type !== "tool-result" || part.result.type !== "content") return part + return { + ...part, + result: { + ...part.result, + value: part.result.value.map((item: ToolContent) => { + if (item.type !== "file") return item + return unsupportedMedia(item.mime, item.name, capabilities) ?? item + }), + }, + } + }), + }), + ) + /** * Builds an outbound model request and captures the tool-call capability that * must remain paired with it. It does not execute the request or mutate @@ -87,7 +127,7 @@ export const layer = Layer.effect( }, providerOptions: { openai: { promptCacheKey } }, system: contextEvent.system, - messages: contextEvent.messages, + messages: unsupportedParts(contextEvent.messages, resolved.capabilities), tools: hookedTools, toolChoice: stepLimitReached ? "none" : undefined, }) diff --git a/packages/core/src/session/runner/model.ts b/packages/core/src/session/runner/model.ts index a8435575c9..efb584fb43 100644 --- a/packages/core/src/session/runner/model.ts +++ b/packages/core/src/session/runner/model.ts @@ -82,6 +82,8 @@ export interface Resolved { readonly model: Model /** Selected catalog identity. Durable records and displays must use this, never the API model id. */ readonly ref: ModelV2.Ref + /** Catalog capabilities used to shape requests before provider lowering. */ + readonly capabilities: ModelV2.Capabilities /** Catalog pricing in dollars per million tokens. */ readonly cost: ModelV2.Info["cost"] } @@ -96,14 +98,22 @@ export class Service extends Context.Service()("@opencode/v2 export const layerWith = (resolve: Interface["resolve"]) => Layer.succeed(Service, Service.of({ resolve })) /** Builds a Resolved whose catalog identity mirrors the route model. Test or embedding seam. */ -export const resolved = (model: Model, variant?: ModelV2.VariantID, cost: ModelV2.Info["cost"] = []): Resolved => ({ +export const resolved = ( + model: Model, + options: { + readonly capabilities: ModelV2.Capabilities + readonly variant?: ModelV2.VariantID + readonly cost: ModelV2.Info["cost"] + }, +): Resolved => ({ model, ref: ModelV2.Ref.make({ id: ModelV2.ID.make(model.id), providerID: ProviderV2.ID.make(model.provider), - ...(variant === undefined ? {} : { variant }), + ...(options.variant === undefined ? {} : { variant: options.variant }), }), - cost, + capabilities: options.capabilities, + cost: options.cost, }) const apiKey = (model: ModelV2.Info, credential?: Credential.Value) => { @@ -359,6 +369,7 @@ const layer = Layer.effect( providerID: selected.providerID, ...(session.model?.variant === undefined ? {} : { variant: session.model.variant }), }), + capabilities: selected.capabilities, cost: selected.cost, } }), diff --git a/packages/core/test/session-compact.test.ts b/packages/core/test/session-compact.test.ts index 479e0ec836..d37cc83c12 100644 --- a/packages/core/test/session-compact.test.ts +++ b/packages/core/test/session-compact.test.ts @@ -49,7 +49,14 @@ const client = Layer.mock(LLMClient.Service)({ generate: () => Effect.die("unused"), }) const config = Layer.mock(Config.Service)({ entries: () => Effect.succeed([]) }) -const models = SessionRunnerModel.layerWith(() => Effect.succeed(SessionRunnerModel.resolved(model))) +const models = SessionRunnerModel.layerWith(() => + Effect.succeed( + SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost: [], + }), + ), +) const locations = Layer.effect( LocationServiceMap.Service, LayerMap.make( diff --git a/packages/core/test/session-compaction.test.ts b/packages/core/test/session-compaction.test.ts index 0634e42224..c58433b9cd 100644 --- a/packages/core/test/session-compaction.test.ts +++ b/packages/core/test/session-compaction.test.ts @@ -68,7 +68,13 @@ const client = Layer.mock(LLMClient.Service)({ }) const config = Layer.mock(Config.Service)({ entries: () => Effect.succeed([]) }) const models = Layer.mock(SessionRunnerModel.Service)({ - resolve: () => Effect.succeed(SessionRunnerModel.resolved(model, undefined, cost)), + resolve: () => + Effect.succeed( + SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost, + }), + ), }) const it = testEffect( AppNodeBuilder.build( diff --git a/packages/core/test/session-generate.test.ts b/packages/core/test/session-generate.test.ts index b8be85433f..879c4cb2b3 100644 --- a/packages/core/test/session-generate.test.ts +++ b/packages/core/test/session-generate.test.ts @@ -65,7 +65,14 @@ const client = Layer.mock(LLMClient.Service)({ return response }), }) -const models = SessionRunnerModel.layerWith(() => Effect.succeed(SessionRunnerModel.resolved(model))) +const models = SessionRunnerModel.layerWith(() => + Effect.succeed( + SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost: [], + }), + ), +) const builtins = Layer.mock(InstructionBuiltIns.Service, { load: () => Effect.succeed( diff --git a/packages/core/test/session-model-request.test.ts b/packages/core/test/session-model-request.test.ts new file mode 100644 index 0000000000..418c9b2419 --- /dev/null +++ b/packages/core/test/session-model-request.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, test } from "bun:test" +import { Message, ToolResultPart } from "@opencode-ai/ai" +import { unsupportedParts } from "@opencode-ai/core/session/model-request" + +const capabilities = (input: string[]) => ({ tools: true, input, output: ["text"] }) + +describe("SessionModelRequest.unsupportedParts", () => { + test("replaces unsupported user media with a visible error", () => { + const messages = unsupportedParts( + [ + Message.user([ + Message.text("Describe this image"), + { type: "media", mediaType: "image/png", data: "aGVsbG8=", filename: "logo.png" }, + ]), + ], + capabilities(["text"]), + ) + + expect(messages[0]?.content).toEqual([ + Message.text("Describe this image"), + Message.text('ERROR: Cannot read "logo.png" (this model does not support image input). Inform the user.'), + ]) + }) + + test("replaces unsupported media nested in tool results", () => { + const messages = unsupportedParts( + [ + Message.tool( + ToolResultPart.make({ + id: "call_1", + name: "read", + result: { + type: "content", + value: [ + { type: "text", text: "Image read successfully" }, + { type: "file", uri: "data:image/png;base64,aGVsbG8=", mime: "image/png", name: "logo.png" }, + ], + }, + }), + ), + ], + capabilities(["text"]), + ) + + expect(messages[0]?.content[0]).toMatchObject({ + type: "tool-result", + result: { + type: "content", + value: [ + { type: "text", text: "Image read successfully" }, + { + type: "text", + text: 'ERROR: Cannot read "logo.png" (this model does not support image input). Inform the user.', + }, + ], + }, + }) + }) + + test("preserves supported media", () => { + const message = Message.user({ type: "media", mediaType: "image/png", data: "aGVsbG8=" }) + expect(unsupportedParts([message], capabilities(["text", "image"]))[0]?.content).toEqual(message.content) + }) +}) diff --git a/packages/core/test/session-runner-recorded.test.ts b/packages/core/test/session-runner-recorded.test.ts index ba4329ca78..d26fa37df4 100644 --- a/packages/core/test/session-runner-recorded.test.ts +++ b/packages/core/test/session-runner-recorded.test.ts @@ -73,7 +73,14 @@ const model = OpenAIChat.route generation: { maxTokens: 20, temperature: 0 }, }) .model({ id: "gpt-4o-mini" }) -const models = SessionRunnerModel.layerWith(() => Effect.succeed(SessionRunnerModel.resolved(model))) +const models = SessionRunnerModel.layerWith(() => + Effect.succeed( + SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost: [], + }), + ), +) const systemContext = Layer.mock(InstructionBuiltIns.Service, { load: () => Effect.succeed(Instructions.empty) }) const instructionContext = Layer.mock(InstructionDiscovery.Service, { load: () => Effect.succeed(Instructions.empty) }) const skillInstructions = Layer.mock(SkillInstructions.Service, { load: () => Effect.succeed(Instructions.empty) }) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 6de3b0d070..1208161827 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -283,10 +283,11 @@ let currentModel = model const models = SessionRunnerModel.layerWith((session) => modelResolveHook.pipe( Effect.as( - SessionRunnerModel.resolved( - session.model?.id === "replacement" ? replacementModel : currentModel, - session.model?.variant, - ), + SessionRunnerModel.resolved(session.model?.id === "replacement" ? replacementModel : currentModel, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost: [], + variant: session.model?.variant, + }), ), ), ) diff --git a/packages/core/test/session-title.test.ts b/packages/core/test/session-title.test.ts index 3263930e1e..cb53c3dcde 100644 --- a/packages/core/test/session-title.test.ts +++ b/packages/core/test/session-title.test.ts @@ -65,7 +65,13 @@ const client = Layer.mock(LLMClient.Service)({ generate: () => Effect.die("unused"), }) const models = Layer.mock(SessionRunnerModel.Service)({ - resolve: () => Effect.succeed(SessionRunnerModel.resolved(model, undefined, cost)), + resolve: () => + Effect.succeed( + SessionRunnerModel.resolved(model, { + capabilities: { tools: true, input: ["text", "image"], output: ["text"] }, + cost, + }), + ), }) const it = testEffect( AppNodeBuilder.build(