From 50dde6a6468a663b0a8109b87a16e26e603910fc Mon Sep 17 00:00:00 2001 From: Ryan Vogel <89211796+R44VC0RP@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:05:22 +0000 Subject: [PATCH] fix(core): restore webfetch image output --- packages/core/src/tool/webfetch.ts | 42 ++++++++++++++--- packages/core/test/tool-webfetch.test.ts | 57 +++++++++++++++++++++--- packages/docs/attachments.mdx | 10 +++-- 3 files changed, 96 insertions(+), 13 deletions(-) diff --git a/packages/core/src/tool/webfetch.ts b/packages/core/src/tool/webfetch.ts index e7a372e4b4..e9d794bf48 100644 --- a/packages/core/src/tool/webfetch.ts +++ b/packages/core/src/tool/webfetch.ts @@ -36,6 +36,14 @@ const Output = Schema.Struct({ contentType: Schema.String, format: Input.fields.format, output: Schema.String, + data: Schema.String.pipe(Schema.optional), + mime: Schema.String.pipe(Schema.optional), +}) +const StructuredOutput = Schema.Struct({ + url: Output.fields.url, + contentType: Output.fields.contentType, + format: Output.fields.format, + output: Output.fields.output, }) type Format = (typeof Input.Type)["format"] @@ -104,7 +112,9 @@ const isTextualMime = (mime: string) => mime === "application/xml" || mime.endsWith("+xml") || mime === "application/javascript" || - mime === "application/x-javascript" + mime === "application/x-javascript" || + mime === "image/svg+xml" || + mime === "image/vnd.fastbidsheet" const convert = (content: string, contentType: string, format: Format) => { if (!contentType.includes("text/html")) return content if (format === "markdown") return convertHTMLToMarkdown(content) @@ -126,7 +136,21 @@ export const Plugin = { description, input: Input, output: Output, - toModelOutput: ({ output }) => [{ type: "text", text: output.output }], + structured: StructuredOutput, + // Image base64 belongs in the native file content only so it is not persisted twice. + toStructuredOutput: ({ output }) => ({ + url: output.url, + contentType: output.contentType, + format: output.format, + output: output.output, + }), + toModelOutput: ({ input, output }) => { + if (output.data === undefined || output.mime === undefined) return [{ type: "text", text: output.output }] + return [ + { type: "text", text: output.output }, + { type: "file", data: output.data, mime: output.mime, name: input.url }, + ] + }, execute: (input, context) => Effect.gen(function* () { yield* Effect.try({ @@ -150,9 +174,7 @@ export const Plugin = { ) const contentType = response.headers["content-type"] || "" const mime = mimeFrom(contentType) - if (isImageAttachment(mime)) - return yield* Effect.fail(new Error(`Unsupported fetched image content type: ${mime}`)) - if (!isTextualMime(mime)) + if (!isTextualMime(mime) && !isImageAttachment(mime)) return yield* Effect.fail(new Error(`Unsupported fetched file content type: ${mime}`)) return { body: yield* collectBody(response), contentType } }).pipe( @@ -161,6 +183,16 @@ export const Plugin = { orElse: () => Effect.fail(new Error("Request timed out")), }), ) + const mime = mimeFrom(contentType) + if (isImageAttachment(mime)) + return { + url: input.url, + contentType, + format: input.format, + output: "Image fetched successfully", + data: Buffer.from(body).toString("base64"), + mime, + } const content = new TextDecoder().decode(body) const output = yield* Effect.try({ try: () => convert(content, contentType, input.format), diff --git a/packages/core/test/tool-webfetch.test.ts b/packages/core/test/tool-webfetch.test.ts index fc38296c9b..aafec9384a 100644 --- a/packages/core/test/tool-webfetch.test.ts +++ b/packages/core/test/tool-webfetch.test.ts @@ -234,14 +234,41 @@ describe("WebFetchTool registration", () => { }), ) - it.effect("keeps images and files unsupported until typed settlement can carry attachments", () => + it.effect("returns images as native media and keeps other files unsupported", () => Effect.gen(function* () { reset() const registry = yield* ToolRegistry.Service - respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "image/png" } })) - expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({ - type: "error", - value: "Unable to fetch https://1.1.1.1/image", + respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "IMAGE/PNG; charset=binary" } })) + expect(yield* settleTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({ + result: { + type: "content", + value: [ + { type: "text", text: "Image fetched successfully" }, + { + type: "file", + uri: "data:image/png;base64,cG5n", + mime: "image/png", + name: "https://1.1.1.1/image", + }, + ], + }, + output: { + structured: { + url: "https://1.1.1.1/image", + contentType: "IMAGE/PNG; charset=binary", + format: "html", + output: "Image fetched successfully", + }, + content: [ + { type: "text", text: "Image fetched successfully" }, + { + type: "file", + uri: "data:image/png;base64,cG5n", + mime: "image/png", + name: "https://1.1.1.1/image", + }, + ], + }, }) respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } })) @@ -252,6 +279,26 @@ describe("WebFetchTool registration", () => { }), ) + it.effect("keeps SVG and FastBid sheets as text instead of image media", () => + Effect.gen(function* () { + reset() + const registry = yield* ToolRegistry.Service + respond = () => + Effect.succeed(new Response("hello", { headers: { "content-type": "image/svg+xml" } })) + expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image.svg", format: "html" }))).toEqual({ + type: "text", + value: "hello", + }) + + respond = () => + Effect.succeed(new Response("table Sheet {}", { headers: { "content-type": "image/vnd.fastbidsheet" } })) + expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/sheet.fbs", format: "text" }))).toEqual({ + type: "text", + value: "table Sheet {}", + }) + }), + ) + it.effect("retries Cloudflare challenges with an honest user agent", () => Effect.gen(function* () { reset() diff --git a/packages/docs/attachments.mdx b/packages/docs/attachments.mdx index 7f97859bf4..6d09e2acbd 100644 --- a/packages/docs/attachments.mdx +++ b/packages/docs/attachments.mdx @@ -17,6 +17,10 @@ and other binary prompt attachments are not currently included in the model request. Some clients may let you select a PDF, but V2 does not yet make that PDF visible to the model. +The built-in `webfetch` tool returns HTTP image responses as image media when +the response uses an `image/*` content type. SVG and FastBid sheet responses +remain text. + Use a model that supports image input before attaching an image. OpenCode passes supported image media to the selected provider, but the provider and @@ -133,9 +137,9 @@ All fields are optional: In the current V2 runtime, these settings apply to image media produced by - the built-in `read` tool. Images attached directly through the TUI, desktop, - web, CLI, or API bypass this normalization. Resize direct attachments before - adding them if the provider requires smaller media. + built-in tools such as `read` and `webfetch`. Images attached directly + through the TUI, desktop, web, CLI, or API bypass this normalization. Resize + direct attachments before adding them if the provider requires smaller media. The `read` tool recognizes PNG, JPEG, GIF, and WebP by their contents and will