fix(core): make V2 reads media-aware and binary-safe (#31038)

packages/core/src/filesystem.ts

@@ -23,9 +23,89 @@ export type ReadInput = typeof ReadInput.Type
 
 export const MAX_READ_LINES = 2_000
 export const MAX_READ_BYTES = 50 * 1024
+export const READ_SAMPLE_BYTES = 4 * 1024
+export const MAX_MEDIA_INGEST_BYTES = 20 * 1024 * 1024
 const MAX_LINE_LENGTH = 2_000
 const MAX_LINE_SUFFIX = `... (line truncated to ${MAX_LINE_LENGTH} chars)`
 
+export class ReadLimitError extends Error {
+  constructor(
+    readonly resource: string,
+    readonly maximumBytes: number,
+  ) {
+    super(`File exceeds ${maximumBytes} byte read limit: ${resource}`)
+    this.name = "ReadLimitError"
+  }
+}
+
+export class BinaryFileError extends Error {
+  constructor(readonly resource: string) {
+    super(`Cannot read binary file: ${resource}`)
+    this.name = "BinaryFileError"
+  }
+}
+
+const BINARY_EXTENSIONS = new Set([
+  ".zip",
+  ".tar",
+  ".gz",
+  ".exe",
+  ".dll",
+  ".so",
+  ".class",
+  ".jar",
+  ".war",
+  ".7z",
+  ".doc",
+  ".docx",
+  ".xls",
+  ".xlsx",
+  ".ppt",
+  ".pptx",
+  ".odt",
+  ".ods",
+  ".odp",
+  ".bin",
+  ".dat",
+  ".obj",
+  ".o",
+  ".a",
+  ".lib",
+  ".wasm",
+  ".pyc",
+  ".pyo",
+])
+
+export const isBinary = (resource: string, bytes: Uint8Array) => {
+  if (BINARY_EXTENSIONS.has(path.extname(resource).toLowerCase())) return true
+  if (bytes.length === 0) return false
+  let nonPrintable = 0
+  for (const byte of bytes) {
+    if (byte === 0) return true
+    if (byte < 9 || (byte > 13 && byte < 32)) nonPrintable++
+  }
+  return nonPrintable / bytes.length > 0.3
+}
+
+const startsWith = (bytes: Uint8Array, prefix: number[]) => prefix.every((value, index) => bytes[index] === value)
+const supportedImageMime = (bytes: Uint8Array) => {
+  if (startsWith(bytes, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) return "image/png"
+  if (startsWith(bytes, [0xff, 0xd8, 0xff])) return "image/jpeg"
+  if (startsWith(bytes, [0x47, 0x49, 0x46, 0x38])) return "image/gif"
+  if (startsWith(bytes, [0x52, 0x49, 0x46, 0x46]) && startsWith(bytes.subarray(8), [0x57, 0x45, 0x42, 0x50]))
+    return "image/webp"
+}
+
+export class MediaIngestLimitError extends Error {
+  constructor(
+    readonly resource: string,
+    readonly maximumBytes: number,
+  ) {
+    super(`Media exceeds ${maximumBytes} byte ingestion limit: ${resource}`)
+    this.name = "MediaIngestLimitError"
+  }
+}
+
 export class TextContent extends Schema.Class<TextContent>("FileSystem.TextContent")({
   type: Schema.Literal("text"),
   content: Schema.String,
@@ -158,7 +238,9 @@ export interface Interface {
   readonly resolveReadPath: (input: ReadInput) => Effect.Effect<ReadPathTarget>
   readonly resolveRead: (input: ReadInput) => Effect.Effect<ReadTarget>
   readonly readResolved: (target: ReadTarget, maximumBytes?: number) => Effect.Effect<Content>
+  readonly readSampleResolved: (target: ReadTarget, maximumBytes: number) => Effect.Effect<Uint8Array>
   readonly readTextPageResolved: (target: ReadTarget, page?: TextPageInput) => Effect.Effect<TextPage>
+  readonly readToolResolved: (target: ReadTarget, page?: TextPageInput) => Effect.Effect<Content | TextPage>
   readonly list: (input?: ListInput) => Effect.Effect<Entry[]>
   /** Select a contained canonical read root without asserting leaf policy. */
   readonly resolveRoot: (input?: ListInput) => Effect.Effect<RootTarget>
@@ -330,15 +412,29 @@ export const layer = Layer.effect(
           if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
           if (info.dev !== target.dev || Option.getOrUndefined(info.ino) !== target.ino)
             return yield* Effect.die(new Error("File changed after permission approval"))
-          if (info.size > maximumBytes)
-            return yield* Effect.die(new Error(`File exceeds ${maximumBytes} byte read limit`))
+          if (info.size > maximumBytes) return yield* Effect.die(new ReadLimitError(target.resource, maximumBytes))
           const bytes = yield* file.readAlloc(maximumBytes + 1).pipe(Effect.orDie)
           if (bytes._tag === "Some" && bytes.value.length > maximumBytes)
-            return yield* Effect.die(new Error(`File exceeds ${maximumBytes} byte read limit`))
+            return yield* Effect.die(new ReadLimitError(target.resource, maximumBytes))
           return yield* content(target, bytes._tag === "Some" ? bytes.value : new Uint8Array())
         }),
       )
     })
+    const readSampleResolved = Effect.fn("FileSystem.readSampleResolved")(function* (
+      target: ReadTarget,
+      maximumBytes: number,
+    ) {
+      return yield* Effect.scoped(
+        Effect.gen(function* () {
+          const file = yield* fs.open(target.real, { flag: "r" }).pipe(Effect.orDie)
+          const info = yield* file.stat.pipe(Effect.orDie)
+          if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
+          if (info.dev !== target.dev || Option.getOrUndefined(info.ino) !== target.ino)
+            return yield* Effect.die(new Error("File changed after permission approval"))
+          return Option.getOrElse(yield* file.readAlloc(maximumBytes).pipe(Effect.orDie), () => new Uint8Array())
+        }),
+      )
+    })
     const readTextPageResolved = Effect.fn("FileSystem.readTextPageResolved")(function* (
       target: ReadTarget,
       page: TextPageInput = {},
@@ -391,7 +487,7 @@ export const layer = Layer.effect(
           while (!done) {
             const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
             if (Option.isNone(chunk)) break
-            if (chunk.value.includes(0)) return yield* Effect.die(new Error("Cannot page binary file"))
+            if (chunk.value.includes(0)) return yield* Effect.die(new BinaryFileError(target.resource))
             let text = decoder.decode(chunk.value, { stream: true })
             while (true) {
               const index = text.indexOf("\n")
@@ -433,6 +529,148 @@ export const layer = Layer.effect(
         }),
       )
     })
+    const readToolResolved = Effect.fn("FileSystem.readToolResolved")(function* (
+      target: ReadTarget,
+      page: TextPageInput = {},
+    ) {
+      return yield* Effect.scoped(
+        Effect.gen(function* () {
+          const file = yield* fs.open(target.real, { flag: "r" }).pipe(Effect.orDie)
+          const info = yield* file.stat.pipe(Effect.orDie)
+          if (info.type !== "File") return yield* Effect.die(new Error("Path is not a file"))
+          if (info.dev !== target.dev || Option.getOrUndefined(info.ino) !== target.ino)
+            return yield* Effect.die(new Error("File changed after permission approval"))
+
+          const first = Option.getOrElse(
+            yield* file.readAlloc(Math.min(64 * 1024, Number(info.size) || READ_SAMPLE_BYTES)).pipe(Effect.orDie),
+            () => new Uint8Array(),
+          )
+          const mime = supportedImageMime(first)
+          if (mime) {
+            if (info.size > MAX_MEDIA_INGEST_BYTES)
+              return yield* Effect.die(new MediaIngestLimitError(target.resource, MAX_MEDIA_INGEST_BYTES))
+            const chunks = [first]
+            let total = first.length
+            while (total <= MAX_MEDIA_INGEST_BYTES) {
+              const chunk = yield* file
+                .readAlloc(Math.min(64 * 1024, MAX_MEDIA_INGEST_BYTES + 1 - total))
+                .pipe(Effect.orDie)
+              if (Option.isNone(chunk)) break
+              chunks.push(chunk.value)
+              total += chunk.value.length
+            }
+            if (total > MAX_MEDIA_INGEST_BYTES)
+              return yield* Effect.die(new MediaIngestLimitError(target.resource, MAX_MEDIA_INGEST_BYTES))
+            return new BinaryContent({
+              type: "binary",
+              content: Buffer.concat(
+                chunks.map((chunk) => Buffer.from(chunk)),
+                total,
+              ).toString("base64"),
+              encoding: "base64",
+              mime,
+            })
+          }
+          if (startsWith(first, [0x25, 0x50, 0x44, 0x46]) || isBinary(target.resource, first))
+            return yield* Effect.die(new BinaryFileError(target.resource))
+
+          const paged = info.size > MAX_READ_BYTES || page.offset !== undefined || page.limit !== undefined
+          if (!paged) {
+            const decoder = new TextDecoder("utf-8", { fatal: true })
+            const text = [yield* Effect.sync(() => decoder.decode(first, { stream: true }))]
+            while (true) {
+              const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
+              if (Option.isNone(chunk)) break
+              if (chunk.value.includes(0)) return yield* Effect.die(new BinaryFileError(target.resource))
+              text.push(yield* Effect.sync(() => decoder.decode(chunk.value, { stream: true })))
+            }
+            text.push(yield* Effect.sync(() => decoder.decode()))
+            return new TextContent({ type: "text", content: text.join(""), mime: FSUtil.mimeType(target.real) })
+          }
+
+          const offset = page.offset ?? 1
+          const limit = Math.min(page.limit ?? MAX_READ_LINES, MAX_READ_LINES)
+          const lines: string[] = []
+          const decoder = new TextDecoder("utf-8", { fatal: true })
+          let pending = ""
+          let discard = false
+          let line = 1
+          let bytes = 0
+          let found = false
+          let truncated = false
+          let next: number | undefined
+
+          const append = (input: string) => {
+            if (line < offset) {
+              line++
+              return
+            }
+            if (lines.length >= limit || bytes >= MAX_READ_BYTES) {
+              truncated = true
+              next ??= line
+              line++
+              return
+            }
+            found = true
+            const text = input.length > MAX_LINE_LENGTH ? input.slice(0, MAX_LINE_LENGTH) + MAX_LINE_SUFFIX : input
+            const size = Buffer.byteLength(text, "utf-8") + (lines.length > 0 ? 1 : 0)
+            if (bytes + size > MAX_READ_BYTES) {
+              truncated = true
+              next ??= line
+              line++
+              return
+            }
+            lines.push(text)
+            bytes += size
+            line++
+          }
+
+          const consume = (chunk: Uint8Array) => {
+            if (chunk.includes(0)) throw new BinaryFileError(target.resource)
+            let text = decoder.decode(chunk, { stream: true })
+            while (true) {
+              const index = text.indexOf("\n")
+              if (index === -1) {
+                if (!discard) {
+                  pending += text
+                  if (pending.length > MAX_LINE_LENGTH) {
+                    pending = pending.slice(0, MAX_LINE_LENGTH + 1)
+                    discard = true
+                  }
+                }
+                break
+              }
+              const current = pending + (discard ? "" : text.slice(0, index))
+              pending = ""
+              discard = false
+              text = text.slice(index + 1)
+              append(current.endsWith("\r") ? current.slice(0, -1) : current)
+            }
+          }
+
+          yield* Effect.sync(() => consume(first))
+          while (true) {
+            const chunk = yield* file.readAlloc(64 * 1024).pipe(Effect.orDie)
+            if (Option.isNone(chunk)) break
+            yield* Effect.sync(() => consume(chunk.value))
+          }
+          const tail = yield* Effect.sync(() => decoder.decode())
+          if (!discard) pending += tail
+          if (pending) append(pending.endsWith("\r") ? pending.slice(0, -1) : pending)
+          if (!found && offset !== 1) return yield* Effect.die(new Error(`Offset ${offset} is out of range`))
+
+          const text = lines.join("\n")
+          return new TextPage({
+            type: "text-page",
+            content: text,
+            mime: FSUtil.mimeType(target.real),
+            offset,
+            truncated,
+            ...(next === undefined ? {} : { next }),
+          })
+        }),
+      )
+    })
     const resolveList = Effect.fn("FileSystem.resolveList")(function* (input: ListInput = {}) {
       const directory = yield* resolve(input.path, input.reference)
       const info = yield* fs.stat(directory.real).pipe(Effect.orDie)
@@ -528,7 +766,9 @@ export const layer = Layer.effect(
       resolveReadPath,
       resolveRead,
       readResolved,
+      readSampleResolved,
       readTextPageResolved,
+      readToolResolved,
       list: Effect.fn("FileSystem.list")(function* (input) {
         return yield* listResolved(yield* resolveList(input))
       }),

packages/core/src/session/compaction.ts

@@ -81,6 +81,13 @@ const estimate = (value: unknown) => Token.estimate(JSON.stringify(value))
 const truncate = (value: string) =>
   value.length <= TOOL_OUTPUT_MAX_CHARS ? value : `${value.slice(0, TOOL_OUTPUT_MAX_CHARS)}\n[truncated]`
 
+export const serializeToolContent = (content: SessionMessage.ToolStateCompleted["content"]) =>
+  content
+    .map((item) =>
+      item.type === "text" ? item.text : `[Attached ${item.mime}${item.name === undefined ? "" : `: ${item.name}`}]`,
+    )
+    .join("\n")
+
 const serialize = (message: SessionMessage.Message) => {
   if (message.type === "user") {
     const files = message.files?.map((file) => `[Attached ${file.mime}: ${file.name ?? file.uri}]`) ?? []
@@ -95,7 +102,7 @@ const serialize = (message: SessionMessage.Message) => {
         if (part.state.status === "completed")
           return [
             `[Assistant tool call]: ${part.name}(${input})`,
-            `[Tool result]: ${truncate(JSON.stringify(part.state.content))}`,
+            `[Tool result]: ${truncate(serializeToolContent(part.state.content))}`,
           ]
         if (part.state.status === "error")
           return [`[Assistant tool call]: ${part.name}(${input})`, `[Tool error]: ${part.state.error.message}`]

packages/core/src/session/runner/publish-llm-event.ts

@@ -350,7 +350,7 @@ export const createLLMEventPublisher = (events: EventV2.Interface, input: Input)
           callID: event.id,
           ...result,
           outputPaths,
-          result: event.result,
+          ...(provider.executed ? { result: event.result } : {}),
           provider,
         })
         return

packages/core/src/tool/read.ts

@@ -1,12 +1,46 @@
 export * as ReadTool from "./read"
 
 import { Tool, ToolFailure } from "@opencode-ai/llm"
+// @ts-ignore Bun's static file import is embedded by `bun build --compile`; some consumers also declare *.wasm.
+import photonWasm from "@silvia-odwyer/photon-node/photon_rs_bg.wasm" with { type: "file" }
 import { Cause, Effect, Layer, Schema } from "effect"
+import path from "node:path"
+import { fileURLToPath } from "node:url"
+import { Config } from "../config"
 import { FileSystem } from "../filesystem"
 import { PermissionV2 } from "../permission"
 import { ToolRegistry } from "./registry"
 
 export const name = "read"
+const SUPPORTED_IMAGE_MIMES = new Set(["image/jpeg", "image/png", "image/gif", "image/webp"])
+const MAX_IMAGE_BASE64_BYTES = 5 * 1024 * 1024
+const MAX_IMAGE_WIDTH = 2_000
+const MAX_IMAGE_HEIGHT = 2_000
+const JPEG_QUALITIES = [80, 85, 70, 55, 40]
+
+class ImageDecodeError extends Error {
+  constructor(readonly resource: string) {
+    super(`Image could not be decoded: ${resource}`)
+    this.name = "ImageDecodeError"
+  }
+}
+
+class ImageSizeError extends Error {
+  constructor(
+    readonly resource: string,
+    readonly width: number,
+    readonly height: number,
+    readonly bytes: number,
+    readonly maxWidth: number,
+    readonly maxHeight: number,
+    readonly maxBytes: number,
+  ) {
+    super(
+      `Image ${resource} is ${width}x${height} with base64 size ${bytes}, exceeding configured limits ${maxWidth}x${maxHeight}/${maxBytes} bytes`,
+    )
+    this.name = "ImageSizeError"
+  }
+}
 const LocationInput = Schema.Struct({
   ...FileSystem.ReadInput.fields,
   offset: FileSystem.ListPageInput.fields.offset.annotate({
@@ -21,15 +55,38 @@ const Success = Schema.Union([FileSystem.Content, FileSystem.TextPage, FileSyste
 
 const definition = Tool.make({
   description:
-    "Read a text or binary file, page through a large UTF-8 text file by line offset, or list a directory page relative to the current location. Absolute paths are accepted only for managed tool-output files.",
+    "Read a text file or supported image, page through a large UTF-8 text file by line offset, or list a directory page relative to the current location. Absolute paths are accepted only for managed tool-output files.",
   parameters: Input,
   success: Success,
+  toStructuredOutput: (output) =>
+    "type" in output && output.type === "binary" && SUPPORTED_IMAGE_MIMES.has(output.mime)
+      ? { type: "media", mime: output.mime }
+      : output,
+  toModelOutput: ({ parameters, output }) => {
+    if (!("type" in output) || output.type !== "binary" || !SUPPORTED_IMAGE_MIMES.has(output.mime)) return []
+    return [
+      { type: "text", text: "Image read successfully" },
+      {
+        type: "file",
+        source: { type: "data", data: output.content },
+        mime: output.mime,
+        name: parameters.path,
+      },
+    ]
+  },
 })
 
 export const layer = Layer.effectDiscard(
   Effect.gen(function* () {
     const registry = yield* ToolRegistry.Service
     const filesystem = yield* FileSystem.Service
+    const config = yield* Config.Service
+    const loadPhoton = yield* Effect.cached(
+      Effect.sync(() => {
+        ;(globalThis as typeof globalThis & { __OPENCODE_PHOTON_WASM_PATH?: string }).__OPENCODE_PHOTON_WASM_PATH =
+          path.isAbsolute(photonWasm) ? photonWasm : fileURLToPath(new URL(photonWasm, import.meta.url))
+      }).pipe(Effect.andThen(() => Effect.promise(() => import("@silvia-odwyer/photon-node")))),
+    )
 
     yield* registry.contribute((editor) =>
       editor.set(name, {
@@ -60,21 +117,118 @@ export const layer = Layer.effectDiscard(
             const final = yield* filesystem.resolveReadPath(input)
             if (final.type !== "file" || final.target.resource !== target.resource || final.target.real !== target.real)
               return yield* Effect.die(new Error("File changed after permission approval"))
-            if (
-              final.target.size > FileSystem.MAX_READ_BYTES ||
-              input.offset !== undefined ||
-              input.limit !== undefined
-            )
-              return yield* filesystem.readTextPageResolved(final.target, { offset: input.offset, limit: input.limit })
-            return yield* filesystem.readResolved(final.target, FileSystem.MAX_READ_BYTES)
+            const content = yield* filesystem.readToolResolved(final.target, {
+              offset: input.offset,
+              limit: input.limit,
+            })
+            if (content.type === "binary" && SUPPORTED_IMAGE_MIMES.has(content.mime)) {
+              const mime = content.mime
+              const base64 = content.content
+              const image = Object.assign(
+                {},
+                ...(yield* config.entries()).flatMap((entry) =>
+                  entry.type === "document" && entry.info.attachments?.image ? [entry.info.attachments.image] : [],
+                ),
+              )
+              const limits = {
+                autoResize: image.auto_resize ?? true,
+                maxWidth: image.max_width ?? MAX_IMAGE_WIDTH,
+                maxHeight: image.max_height ?? MAX_IMAGE_HEIGHT,
+                maxBase64Bytes: image.max_base64_bytes ?? MAX_IMAGE_BASE64_BYTES,
+              }
+              const photon = yield* loadPhoton
+              const decoded = yield* Effect.try({
+                try: () => photon.PhotonImage.new_from_byteslice(Buffer.from(base64, "base64")),
+                catch: () => new ImageDecodeError(final.target.resource),
+              })
+              try {
+                const width = decoded.get_width()
+                const height = decoded.get_height()
+                const bytes = Buffer.byteLength(base64, "utf-8")
+                if (width <= limits.maxWidth && height <= limits.maxHeight && bytes <= limits.maxBase64Bytes)
+                  return new FileSystem.BinaryContent({ type: "binary", content: base64, encoding: "base64", mime })
+                if (!limits.autoResize)
+                  return yield* Effect.die(
+                    new ImageSizeError(
+                      final.target.resource,
+                      width,
+                      height,
+                      bytes,
+                      limits.maxWidth,
+                      limits.maxHeight,
+                      limits.maxBase64Bytes,
+                    ),
+                  )
+                const scale = Math.min(1, limits.maxWidth / width, limits.maxHeight / height)
+                const sizes = Array.from({ length: 32 }).reduce<Array<{ width: number; height: number }>>((acc) => {
+                  const previous = acc.at(-1) ?? {
+                    width: Math.max(1, Math.round(width * scale)),
+                    height: Math.max(1, Math.round(height * scale)),
+                  }
+                  const next =
+                    acc.length === 0
+                      ? previous
+                      : {
+                          width: previous.width === 1 ? 1 : Math.max(1, Math.floor(previous.width * 0.75)),
+                          height: previous.height === 1 ? 1 : Math.max(1, Math.floor(previous.height * 0.75)),
+                        }
+                  return acc.some((item) => item.width === next.width && item.height === next.height)
+                    ? acc
+                    : [...acc, next]
+                }, [])
+                for (const size of sizes) {
+                  const resized = photon.resize(decoded, size.width, size.height, photon.SamplingFilter.Lanczos3)
+                  try {
+                    const candidate = [
+                      { content: Buffer.from(resized.get_bytes()).toString("base64"), mime: "image/png" },
+                      ...JPEG_QUALITIES.map((quality) => ({
+                        content: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64"),
+                        mime: "image/jpeg",
+                      })),
+                    ].find((item) => Buffer.byteLength(item.content, "utf-8") <= limits.maxBase64Bytes)
+                    if (candidate)
+                      return new FileSystem.BinaryContent({
+                        type: "binary",
+                        content: candidate.content,
+                        encoding: "base64",
+                        mime: candidate.mime,
+                      })
+                  } finally {
+                    resized.free()
+                  }
+                }
+                return yield* Effect.die(
+                  new ImageSizeError(
+                    final.target.resource,
+                    width,
+                    height,
+                    bytes,
+                    limits.maxWidth,
+                    limits.maxHeight,
+                    limits.maxBase64Bytes,
+                  ),
+                )
+              } finally {
+                decoded.free()
+              }
+            }
+            if (content.type === "binary")
+              return yield* Effect.die(new FileSystem.BinaryFileError(final.target.resource))
+            return content
           }).pipe(
             Effect.catchCause((cause) =>
-              Effect.fail(
-                new ToolFailure({
-                  message: `Unable to read ${input.path}`,
-                  error: Cause.squash(cause),
-                }),
-              ),
+              Effect.gen(function* () {
+                const error = Cause.squash(cause)
+                const message =
+                  error instanceof FileSystem.BinaryFileError ||
+                  error instanceof FileSystem.ReadLimitError ||
+                  error instanceof FileSystem.MediaIngestLimitError ||
+                  error instanceof ImageDecodeError ||
+                  error instanceof ImageSizeError
+                    ? error.message
+                    : `Unable to read ${input.path}`
+                return yield* new ToolFailure({ message, error })
+              }),
             ),
           )
         },
@@ -85,5 +239,6 @@ export const layer = Layer.effectDiscard(
 export const locationLayer = layer.pipe(
   Layer.provideMerge(ToolRegistry.defaultLayer),
   Layer.provideMerge(FileSystem.locationLayer),
+  Layer.provideMerge(Config.locationLayer),
   Layer.provideMerge(PermissionV2.locationLayer),
 )