fix(core): filter unsupported media inputs (#38145)

Co-authored-by: Aiden Cline <63023139+rekram1-node@users.noreply.github.com>

packages/core/src/session/model-request.ts

@@ -1,10 +1,11 @@
 export * as SessionModelRequest from "./model-request"
 
-import { LLM, Message, SystemPart, type LLMRequest } from "@opencode-ai/ai"
+import { LLM, Message, SystemPart, type LLMRequest, type ToolContent } from "@opencode-ai/ai"
 import { SessionError } from "@opencode-ai/schema/session-error"
 import { Context, Effect, Layer } from "effect"
 import { makeLocationNode } from "@opencode-ai/util/effect/app-node"
 import { Client } from "@opencode-ai/util/client"
+import { ModelV2 } from "../model"
 import { PluginHooks } from "../plugin/hooks"
 import { ToolRegistry } from "../tool/registry"
 import { SessionContext } from "./context"
@@ -27,6 +28,45 @@ interface PrepareInput {
   readonly step: number
 }
 
+const mimeToModality = (mime: string) => {
+  if (mime.startsWith("image/")) return "image"
+  if (mime.startsWith("audio/")) return "audio"
+  if (mime.startsWith("video/")) return "video"
+  if (mime === "application/pdf") return "pdf"
+}
+
+const unsupportedMedia = (mime: string, name: string | undefined, capabilities: ModelV2.Capabilities) => {
+  const modality = mimeToModality(mime)
+  if (!modality || capabilities.input.some((item) => item.startsWith(modality))) return
+  return {
+    type: "text" as const,
+    text: `ERROR: Cannot read ${name ? `"${name}"` : modality} (this model does not support ${modality} input). Inform the user.`,
+  }
+}
+
+export const unsupportedParts = (messages: LLMRequest["messages"], capabilities: ModelV2.Capabilities) =>
+  messages.map((message) =>
+    Message.make({
+      ...message,
+      content: message.content.map((part) => {
+        if (part.type === "media") {
+          return unsupportedMedia(part.mediaType, part.filename, capabilities) ?? part
+        }
+        if (part.type !== "tool-result" || part.result.type !== "content") return part
+        return {
+          ...part,
+          result: {
+            ...part.result,
+            value: part.result.value.map((item: ToolContent) => {
+              if (item.type !== "file") return item
+              return unsupportedMedia(item.mime, item.name, capabilities) ?? item
+            }),
+          },
+        }
+      }),
+    }),
+  )
+
 /**
  * Builds an outbound model request and captures the tool-call capability that
  * must remain paired with it. It does not execute the request or mutate
@@ -87,7 +127,7 @@ export const layer = Layer.effect(
         },
         providerOptions: { openai: { promptCacheKey } },
         system: contextEvent.system,
-        messages: contextEvent.messages,
+        messages: unsupportedParts(contextEvent.messages, resolved.capabilities),
         tools: hookedTools,
         toolChoice: stepLimitReached ? "none" : undefined,
       })

packages/core/src/session/runner/model.ts

@@ -82,6 +82,8 @@ export interface Resolved {
   readonly model: Model
   /** Selected catalog identity. Durable records and displays must use this, never the API model id. */
   readonly ref: ModelV2.Ref
+  /** Catalog capabilities used to shape requests before provider lowering. */
+  readonly capabilities: ModelV2.Capabilities
   /** Catalog pricing in dollars per million tokens. */
   readonly cost: ModelV2.Info["cost"]
 }
@@ -96,14 +98,22 @@ export class Service extends Context.Service<Service, Interface>()("@opencode/v2
 export const layerWith = (resolve: Interface["resolve"]) => Layer.succeed(Service, Service.of({ resolve }))
 
 /** Builds a Resolved whose catalog identity mirrors the route model. Test or embedding seam. */
-export const resolved = (model: Model, variant?: ModelV2.VariantID, cost: ModelV2.Info["cost"] = []): Resolved => ({
+export const resolved = (
+  model: Model,
+  options: {
+    readonly capabilities: ModelV2.Capabilities
+    readonly variant?: ModelV2.VariantID
+    readonly cost: ModelV2.Info["cost"]
+  },
+): Resolved => ({
   model,
   ref: ModelV2.Ref.make({
     id: ModelV2.ID.make(model.id),
     providerID: ProviderV2.ID.make(model.provider),
-    ...(variant === undefined ? {} : { variant }),
+    ...(options.variant === undefined ? {} : { variant: options.variant }),
   }),
-  cost,
+  capabilities: options.capabilities,
+  cost: options.cost,
 })
 
 const apiKey = (model: ModelV2.Info, credential?: Credential.Value) => {
@@ -359,6 +369,7 @@ const layer = Layer.effect(
             providerID: selected.providerID,
             ...(session.model?.variant === undefined ? {} : { variant: session.model.variant }),
           }),
+          capabilities: selected.capabilities,
           cost: selected.cost,
         }
       }),