fix(core): restore webfetch image output
This commit is contained in:
parent
592ef7433a
commit
50dde6a646
3 changed files with 96 additions and 13 deletions
|
|
@ -36,6 +36,14 @@ const Output = Schema.Struct({
|
|||
contentType: Schema.String,
|
||||
format: Input.fields.format,
|
||||
output: Schema.String,
|
||||
data: Schema.String.pipe(Schema.optional),
|
||||
mime: Schema.String.pipe(Schema.optional),
|
||||
})
|
||||
const StructuredOutput = Schema.Struct({
|
||||
url: Output.fields.url,
|
||||
contentType: Output.fields.contentType,
|
||||
format: Output.fields.format,
|
||||
output: Output.fields.output,
|
||||
})
|
||||
|
||||
type Format = (typeof Input.Type)["format"]
|
||||
|
|
@ -104,7 +112,9 @@ const isTextualMime = (mime: string) =>
|
|||
mime === "application/xml" ||
|
||||
mime.endsWith("+xml") ||
|
||||
mime === "application/javascript" ||
|
||||
mime === "application/x-javascript"
|
||||
mime === "application/x-javascript" ||
|
||||
mime === "image/svg+xml" ||
|
||||
mime === "image/vnd.fastbidsheet"
|
||||
const convert = (content: string, contentType: string, format: Format) => {
|
||||
if (!contentType.includes("text/html")) return content
|
||||
if (format === "markdown") return convertHTMLToMarkdown(content)
|
||||
|
|
@ -126,7 +136,21 @@ export const Plugin = {
|
|||
description,
|
||||
input: Input,
|
||||
output: Output,
|
||||
toModelOutput: ({ output }) => [{ type: "text", text: output.output }],
|
||||
structured: StructuredOutput,
|
||||
// Image base64 belongs in the native file content only so it is not persisted twice.
|
||||
toStructuredOutput: ({ output }) => ({
|
||||
url: output.url,
|
||||
contentType: output.contentType,
|
||||
format: output.format,
|
||||
output: output.output,
|
||||
}),
|
||||
toModelOutput: ({ input, output }) => {
|
||||
if (output.data === undefined || output.mime === undefined) return [{ type: "text", text: output.output }]
|
||||
return [
|
||||
{ type: "text", text: output.output },
|
||||
{ type: "file", data: output.data, mime: output.mime, name: input.url },
|
||||
]
|
||||
},
|
||||
execute: (input, context) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.try({
|
||||
|
|
@ -150,9 +174,7 @@ export const Plugin = {
|
|||
)
|
||||
const contentType = response.headers["content-type"] || ""
|
||||
const mime = mimeFrom(contentType)
|
||||
if (isImageAttachment(mime))
|
||||
return yield* Effect.fail(new Error(`Unsupported fetched image content type: ${mime}`))
|
||||
if (!isTextualMime(mime))
|
||||
if (!isTextualMime(mime) && !isImageAttachment(mime))
|
||||
return yield* Effect.fail(new Error(`Unsupported fetched file content type: ${mime}`))
|
||||
return { body: yield* collectBody(response), contentType }
|
||||
}).pipe(
|
||||
|
|
@ -161,6 +183,16 @@ export const Plugin = {
|
|||
orElse: () => Effect.fail(new Error("Request timed out")),
|
||||
}),
|
||||
)
|
||||
const mime = mimeFrom(contentType)
|
||||
if (isImageAttachment(mime))
|
||||
return {
|
||||
url: input.url,
|
||||
contentType,
|
||||
format: input.format,
|
||||
output: "Image fetched successfully",
|
||||
data: Buffer.from(body).toString("base64"),
|
||||
mime,
|
||||
}
|
||||
const content = new TextDecoder().decode(body)
|
||||
const output = yield* Effect.try({
|
||||
try: () => convert(content, contentType, input.format),
|
||||
|
|
|
|||
|
|
@ -234,14 +234,41 @@ describe("WebFetchTool registration", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps images and files unsupported until typed settlement can carry attachments", () =>
|
||||
it.effect("returns images as native media and keeps other files unsupported", () =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
const registry = yield* ToolRegistry.Service
|
||||
respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "image/png" } }))
|
||||
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({
|
||||
type: "error",
|
||||
value: "Unable to fetch https://1.1.1.1/image",
|
||||
respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "IMAGE/PNG; charset=binary" } }))
|
||||
expect(yield* settleTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({
|
||||
result: {
|
||||
type: "content",
|
||||
value: [
|
||||
{ type: "text", text: "Image fetched successfully" },
|
||||
{
|
||||
type: "file",
|
||||
uri: "data:image/png;base64,cG5n",
|
||||
mime: "image/png",
|
||||
name: "https://1.1.1.1/image",
|
||||
},
|
||||
],
|
||||
},
|
||||
output: {
|
||||
structured: {
|
||||
url: "https://1.1.1.1/image",
|
||||
contentType: "IMAGE/PNG; charset=binary",
|
||||
format: "html",
|
||||
output: "Image fetched successfully",
|
||||
},
|
||||
content: [
|
||||
{ type: "text", text: "Image fetched successfully" },
|
||||
{
|
||||
type: "file",
|
||||
uri: "data:image/png;base64,cG5n",
|
||||
mime: "image/png",
|
||||
name: "https://1.1.1.1/image",
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } }))
|
||||
|
|
@ -252,6 +279,26 @@ describe("WebFetchTool registration", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("keeps SVG and FastBid sheets as text instead of image media", () =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
const registry = yield* ToolRegistry.Service
|
||||
respond = () =>
|
||||
Effect.succeed(new Response("<svg><text>hello</text></svg>", { headers: { "content-type": "image/svg+xml" } }))
|
||||
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image.svg", format: "html" }))).toEqual({
|
||||
type: "text",
|
||||
value: "<svg><text>hello</text></svg>",
|
||||
})
|
||||
|
||||
respond = () =>
|
||||
Effect.succeed(new Response("table Sheet {}", { headers: { "content-type": "image/vnd.fastbidsheet" } }))
|
||||
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/sheet.fbs", format: "text" }))).toEqual({
|
||||
type: "text",
|
||||
value: "table Sheet {}",
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("retries Cloudflare challenges with an honest user agent", () =>
|
||||
Effect.gen(function* () {
|
||||
reset()
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@ and other binary prompt attachments are not currently included in the model
|
|||
request. Some clients may let you select a PDF, but V2 does not yet make that
|
||||
PDF visible to the model.
|
||||
|
||||
The built-in `webfetch` tool returns HTTP image responses as image media when
|
||||
the response uses an `image/*` content type. SVG and FastBid sheet responses
|
||||
remain text.
|
||||
|
||||
<Warning>
|
||||
Use a model that supports image input before attaching an image. OpenCode
|
||||
passes supported image media to the selected provider, but the provider and
|
||||
|
|
@ -133,9 +137,9 @@ All fields are optional:
|
|||
|
||||
<Note>
|
||||
In the current V2 runtime, these settings apply to image media produced by
|
||||
the built-in `read` tool. Images attached directly through the TUI, desktop,
|
||||
web, CLI, or API bypass this normalization. Resize direct attachments before
|
||||
adding them if the provider requires smaller media.
|
||||
built-in tools such as `read` and `webfetch`. Images attached directly
|
||||
through the TUI, desktop, web, CLI, or API bypass this normalization. Resize
|
||||
direct attachments before adding them if the provider requires smaller media.
|
||||
</Note>
|
||||
|
||||
The `read` tool recognizes PNG, JPEG, GIF, and WebP by their contents and will
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue