fix(core): filter unsupported media inputs (#38145)

Co-authored-by: Aiden Cline <63023139+rekram1-node@users.noreply.github.com>

packages/core/src/session/model-request.ts

@@ -1,10 +1,11 @@
 export * as SessionModelRequest from "./model-request"
 
-import { LLM, Message, SystemPart, type LLMRequest } from "@opencode-ai/ai"
+import { LLM, Message, SystemPart, type LLMRequest, type ToolContent } from "@opencode-ai/ai"
 import { SessionError } from "@opencode-ai/schema/session-error"
 import { Context, Effect, Layer } from "effect"
 import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
 import { Client } from "@opencode-ai/util/client"
+import { ModelV2 } from "../model"
 import { PluginHooks } from "../plugin/hooks"
 import { ToolRegistry } from "../tool/registry"
 import { SessionContext } from "./context"
@@ -27,6 +28,45 @@ interface PrepareInput {
   readonly step: number
 }
 
+const mimeToModality = (mime: string) => {
+  if (mime.startsWith("image/")) return "image"
+  if (mime.startsWith("audio/")) return "audio"
+  if (mime.startsWith("video/")) return "video"
+  if (mime === "application/pdf") return "pdf"
+}
+
+const unsupportedMedia = (mime: string, name: string | undefined, capabilities: ModelV2.Capabilities) => {
+  const modality = mimeToModality(mime)
+  if (!modality || capabilities.input.some((item) => item.startsWith(modality))) return
+  return {
+    type: "text" as const,
+    text: `ERROR: Cannot read ${name ? `"${name}"` : modality} (this model does not support ${modality} input). Inform the user.`,
+  }
+}
+
+export const unsupportedParts = (messages: LLMRequest["messages"], capabilities: ModelV2.Capabilities) =>
+  messages.map((message) =>
+    Message.make({
+      ...message,
+      content: message.content.map((part) => {
+        if (part.type === "media") {
+          return unsupportedMedia(part.mediaType, part.filename, capabilities) ?? part
+        }
+        if (part.type !== "tool-result" || part.result.type !== "content") return part
+        return {
+          ...part,
+          result: {
+            ...part.result,
+            value: part.result.value.map((item: ToolContent) => {
+              if (item.type !== "file") return item
+              return unsupportedMedia(item.mime, item.name, capabilities) ?? item
+            }),
+          },
+        }
+      }),
+    }),
+  )
+
 /**
  * Builds an outbound model request and captures the tool-call capability that
  * must remain paired with it. It does not execute the request or mutate
@@ -87,7 +127,7 @@ export const layer = Layer.effect(
         },
         providerOptions: { openai: { promptCacheKey } },
         system: contextEvent.system,
-        messages: contextEvent.messages,
+        messages: unsupportedParts(contextEvent.messages, resolved.capabilities),
         tools: hookedTools,
         toolChoice: stepLimitReached ? "none" : undefined,
       })

packages/core/src/session/runner/model.ts

@@ -82,6 +82,8 @@ export interface Resolved {
   readonly model: Model
   /** Selected catalog identity. Durable records and displays must use this, never the API model id. */
   readonly ref: ModelV2.Ref
+  /** Catalog capabilities used to shape requests before provider lowering. */
+  readonly capabilities: ModelV2.Capabilities
   /** Catalog pricing in dollars per million tokens. */
   readonly cost: ModelV2.Info["cost"]
 }
@@ -96,14 +98,22 @@ export class Service extends Context.Service<Service, Interface>()("@opencode/v2
 export const layerWith = (resolve: Interface["resolve"]) => Layer.succeed(Service, Service.of({ resolve }))
 
 /** Builds a Resolved whose catalog identity mirrors the route model. Test or embedding seam. */
-export const resolved = (model: Model, variant?: ModelV2.VariantID, cost: ModelV2.Info["cost"] = []): Resolved => ({
+export const resolved = (
+  model: Model,
+  options: {
+    readonly capabilities: ModelV2.Capabilities
+    readonly variant?: ModelV2.VariantID
+    readonly cost: ModelV2.Info["cost"]
+  },
+): Resolved => ({
   model,
   ref: ModelV2.Ref.make({
     id: ModelV2.ID.make(model.id),
     providerID: ProviderV2.ID.make(model.provider),
-    ...(variant === undefined ? {} : { variant }),
+    ...(options.variant === undefined ? {} : { variant: options.variant }),
   }),
-  cost,
+  capabilities: options.capabilities,
+  cost: options.cost,
 })
 
 const apiKey = (model: ModelV2.Info, credential?: Credential.Value) => {
@@ -359,6 +369,7 @@ const layer = Layer.effect(
             providerID: selected.providerID,
             ...(session.model?.variant === undefined ? {} : { variant: session.model.variant }),
           }),
+          capabilities: selected.capabilities,
           cost: selected.cost,
         }
       }),

packages/core/test/session-compact.test.ts

@@ -49,7 +49,14 @@ const client = Layer.mock(LLMClient.Service)({
   generate: () => Effect.die("unused"),
 })
 const config = Layer.mock(Config.Service)({ entries: () => Effect.succeed([]) })
-const models = SessionRunnerModel.layerWith(() => Effect.succeed(SessionRunnerModel.resolved(model)))
+const models = SessionRunnerModel.layerWith(() =>
+  Effect.succeed(
+    SessionRunnerModel.resolved(model, {
+      capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
+      cost: [],
+    }),
+  ),
+)
 const locations = Layer.effect(
   LocationServiceMap.Service,
   LayerMap.make(

packages/core/test/session-compaction.test.ts

@@ -68,7 +68,13 @@ const client = Layer.mock(LLMClient.Service)({
 })
 const config = Layer.mock(Config.Service)({ entries: () => Effect.succeed([]) })
 const models = Layer.mock(SessionRunnerModel.Service)({
-  resolve: () => Effect.succeed(SessionRunnerModel.resolved(model, undefined, cost)),
+  resolve: () =>
+    Effect.succeed(
+      SessionRunnerModel.resolved(model, {
+        capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
+        cost,
+      }),
+    ),
 })
 const it = testEffect(
   AppNodeBuilder.build(

packages/core/test/session-generate.test.ts

@@ -65,7 +65,14 @@ const client = Layer.mock(LLMClient.Service)({
       return response
     }),
 })
-const models = SessionRunnerModel.layerWith(() => Effect.succeed(SessionRunnerModel.resolved(model)))
+const models = SessionRunnerModel.layerWith(() =>
+  Effect.succeed(
+    SessionRunnerModel.resolved(model, {
+      capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
+      cost: [],
+    }),
+  ),
+)
 const builtins = Layer.mock(InstructionBuiltIns.Service, {
   load: () =>
     Effect.succeed(

packages/core/test/session-model-request.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, test } from "bun:test"
+import { Message, ToolResultPart } from "@opencode-ai/ai"
+import { unsupportedParts } from "@opencode-ai/core/session/model-request"
+
+const capabilities = (input: string[]) => ({ tools: true, input, output: ["text"] })
+
+describe("SessionModelRequest.unsupportedParts", () => {
+  test("replaces unsupported user media with a visible error", () => {
+    const messages = unsupportedParts(
+      [
+        Message.user([
+          Message.text("Describe this image"),
+          { type: "media", mediaType: "image/png", data: "aGVsbG8=", filename: "logo.png" },
+        ]),
+      ],
+      capabilities(["text"]),
+    )
+
+    expect(messages[0]?.content).toEqual([
+      Message.text("Describe this image"),
+      Message.text('ERROR: Cannot read "logo.png" (this model does not support image input). Inform the user.'),
+    ])
+  })
+
+  test("replaces unsupported media nested in tool results", () => {
+    const messages = unsupportedParts(
+      [
+        Message.tool(
+          ToolResultPart.make({
+            id: "call_1",
+            name: "read",
+            result: {
+              type: "content",
+              value: [
+                { type: "text", text: "Image read successfully" },
+                { type: "file", uri: "data:image/png;base64,aGVsbG8=", mime: "image/png", name: "logo.png" },
+              ],
+            },
+          }),
+        ),
+      ],
+      capabilities(["text"]),
+    )
+
+    expect(messages[0]?.content[0]).toMatchObject({
+      type: "tool-result",
+      result: {
+        type: "content",
+        value: [
+          { type: "text", text: "Image read successfully" },
+          {
+            type: "text",
+            text: 'ERROR: Cannot read "logo.png" (this model does not support image input). Inform the user.',
+          },
+        ],
+      },
+    })
+  })
+
+  test("preserves supported media", () => {
+    const message = Message.user({ type: "media", mediaType: "image/png", data: "aGVsbG8=" })
+    expect(unsupportedParts([message], capabilities(["text", "image"]))[0]?.content).toEqual(message.content)
+  })
+})

packages/core/test/session-runner-recorded.test.ts

@@ -73,7 +73,14 @@ const model = OpenAIChat.route
     generation: { maxTokens: 20, temperature: 0 },
   })
   .model({ id: "gpt-4o-mini" })
-const models = SessionRunnerModel.layerWith(() => Effect.succeed(SessionRunnerModel.resolved(model)))
+const models = SessionRunnerModel.layerWith(() =>
+  Effect.succeed(
+    SessionRunnerModel.resolved(model, {
+      capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
+      cost: [],
+    }),
+  ),
+)
 const systemContext = Layer.mock(InstructionBuiltIns.Service, { load: () => Effect.succeed(Instructions.empty) })
 const instructionContext = Layer.mock(InstructionDiscovery.Service, { load: () => Effect.succeed(Instructions.empty) })
 const skillInstructions = Layer.mock(SkillInstructions.Service, { load: () => Effect.succeed(Instructions.empty) })

packages/core/test/session-runner.test.ts

@@ -283,10 +283,11 @@ let currentModel = model
 const models = SessionRunnerModel.layerWith((session) =>
   modelResolveHook.pipe(
     Effect.as(
-      SessionRunnerModel.resolved(
-        session.model?.id === "replacement" ? replacementModel : currentModel,
-        session.model?.variant,
-      ),
+      SessionRunnerModel.resolved(session.model?.id === "replacement" ? replacementModel : currentModel, {
+        capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
+        cost: [],
+        variant: session.model?.variant,
+      }),
     ),
   ),
 )

packages/core/test/session-title.test.ts

@@ -65,7 +65,13 @@ const client = Layer.mock(LLMClient.Service)({
   generate: () => Effect.die("unused"),
 })
 const models = Layer.mock(SessionRunnerModel.Service)({
-  resolve: () => Effect.succeed(SessionRunnerModel.resolved(model, undefined, cost)),
+  resolve: () =>
+    Effect.succeed(
+      SessionRunnerModel.resolved(model, {
+        capabilities: { tools: true, input: ["text", "image"], output: ["text"] },
+        cost,
+      }),
+    ),
 })
 const it = testEffect(
   AppNodeBuilder.build(