feat(core): normalize tool and attachment images at settlement (#37141)

packages/core/src/tool/read.ts

@@ -6,7 +6,6 @@ import { ToolFailure } from "@opencode-ai/ai"
 import { Effect, Schema } from "effect"
 import { FileSystem } from "../filesystem"
 import { FSUtil } from "../fs-util"
-import { Image } from "../image"
 import { Location } from "../location"
 import { LocationMutation } from "../location-mutation"
 import { PermissionV2 } from "../permission"
@@ -35,7 +34,6 @@ export const Plugin = {
   effect: Effect.fn("ReadTool.Plugin")(function* (ctx: PluginContext) {
     const reader = yield* ReadToolFileSystem.Service
     const mutation = yield* LocationMutation.Service
-    const image = yield* Image.Service
     const permission = yield* PermissionV2.Service
     const sessionInstructions = yield* SessionInstructions.Service
     const fs = yield* FSUtil.Service
@@ -50,6 +48,12 @@ export const Plugin = {
               "Read a text file or supported image, page through a large UTF-8 text file by line offset, or list a directory page. Relative paths resolve from the current location; absolute paths inside it are accepted, while external absolute paths require external_directory approval.",
             input: Input,
             output: Output,
+            structured: Schema.toEncoded(Output),
+            // Image base64 reaches the model through content items (normalized generically
+            // at tool settlement); persisting a second copy in structured would store the
+            // original unresized bytes in the message row.
+            toStructuredOutput: ({ output }) =>
+              "encoding" in output && output.encoding === "base64" ? { ...output, content: "" } : output,
             toModelOutput: ({ input, output }) => {
               if (!("encoding" in output) || output.encoding !== "base64" || !SUPPORTED_IMAGE_MIMES.has(output.mime))
                 return []
@@ -117,21 +121,14 @@ export const Plugin = {
                   Effect.catch(() => Effect.void),
                   Effect.catchDefect(() => Effect.void),
                 )
-                if ("encoding" in content && content.encoding === "base64" && SUPPORTED_IMAGE_MIMES.has(content.mime)) {
-                  return yield* image
-                    .normalize(resource, { ...content, encoding: "base64" })
-                    .pipe(Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)))
-                }
-                if ("encoding" in content && content.encoding === "base64")
+                if ("encoding" in content && content.encoding === "base64" && !SUPPORTED_IMAGE_MIMES.has(content.mime))
                   return yield* Effect.fail(new ReadToolFileSystem.BinaryFileError({ resource }))
                 return content
               }).pipe(
                 Effect.mapError((error) => {
                   const message =
                     error instanceof ReadToolFileSystem.BinaryFileError ||
-                    error instanceof ReadToolFileSystem.MediaIngestLimitError ||
-                    error instanceof Image.DecodeError ||
-                    error instanceof Image.SizeError
+                    error instanceof ReadToolFileSystem.MediaIngestLimitError
                       ? error.message
                       : `Unable to read ${input.path}`
                   return new ToolFailure({ message, error })

packages/core/src/tool/registry.ts

@@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry"
 import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai"
 import { Context, Effect, Layer, Scope } from "effect"
 import type { AgentV2 } from "../agent"
+import { Image } from "../image"
 import { PermissionV2 } from "../permission"
 import { SessionMessage } from "../session/message"
 import { SessionSchema } from "../session/schema"
@@ -57,6 +58,40 @@ const registryLayer = Layer.effect(
   Effect.gen(function* () {
     const resources = yield* ToolOutputStore.Service
     const toolHooks = yield* ToolHooks.Service
+    const image = yield* Image.Service
+
+    type NormalizedItem = ToolOutput["content"][number] | "decode" | "size"
+    const normalizeImages = Effect.fn("ToolRegistry.normalizeImages")(function* (content: ToolOutput["content"]) {
+      const normalized = yield* Effect.forEach(content, (item): Effect.Effect<NormalizedItem> => {
+        if (item.type !== "file" || !item.mime.startsWith("image/")) return Effect.succeed(item)
+        // RFC 2397 permits parameters between the mime and ";base64".
+        const base64 = /^data:[^,]*;base64,(.*)$/s.exec(item.uri)?.[1]
+        if (base64 === undefined) return Effect.succeed(item)
+        const resource = item.name ?? `${item.mime} tool output`
+        return image
+          .normalize(resource, { uri: resource, content: base64, encoding: "base64", mime: item.mime })
+          .pipe(
+            Effect.map((result) => ({
+              ...item,
+              uri: `data:${result.mime};base64,${result.content}`,
+              mime: result.mime,
+            })),
+            Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(item)),
+            Effect.catchTag("Image.DecodeError", () => Effect.succeed("decode" as const)),
+            Effect.catchTag("Image.SizeError", () => Effect.succeed("size" as const)),
+          )
+      })
+      const note = (reason: "decode" | "size", text: string) => {
+        const count = normalized.filter((item) => item === reason).length
+        if (count === 0) return []
+        return [{ type: "text" as const, text: `[${count} image${count === 1 ? "" : "s"} omitted: ${text}]` }]
+      }
+      return [
+        ...normalized.filter((item) => typeof item !== "string"),
+        ...note("decode", "could not be decoded."),
+        ...note("size", "could not be resized below the image size limit."),
+      ]
+    })
     type Registration = {
       readonly tool: AnyTool
       readonly name: string
@@ -84,10 +119,11 @@ const registryLayer = Layer.effect(
           agent: input.agent,
           messageID: input.messageID,
           callID: input.call.id,
-          progress: (update) =>
-            input.progress?.({
-              structured: update.structured,
-              content: (update.content ?? []).map((part) =>
+          progress: (update) => {
+            const progress = input.progress
+            if (!progress) return Effect.void
+            return normalizeImages(
+              (update.content ?? []).map((part) =>
                 part.type === "text"
                   ? { type: "text" as const, text: part.text }
                   : {
@@ -97,7 +133,8 @@ const registryLayer = Layer.effect(
                       name: part.name,
                     },
               ),
-            }) ?? Effect.void,
+            ).pipe(Effect.flatMap((content) => progress({ structured: update.structured, content })))
+          },
         },
       ).pipe(
         Effect.map((output) => ({ output })),
@@ -115,7 +152,7 @@ const registryLayer = Layer.effect(
         const bounded = yield* resources.bound({
           sessionID: input.sessionID,
           callID: input.call.id,
-          output: pending.output,
+          output: { structured: pending.output.structured, content: yield* normalizeImages(pending.output.content) },
         })
         const result = ToolOutput.toResultValue(bounded.output)
         settlement =
@@ -232,11 +269,11 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) {
 export const node = makeLocationNode({
   service: Service,
   layer,
-  deps: [ToolOutputStore.node, ToolHooks.node],
+  deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
 })
 
 export const toolsNode = makeLocationNode({
   service: Tools.Service,
   layer,
-  deps: [ToolOutputStore.node, ToolHooks.node],
+  deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
 })