feat(core): normalize tool and attachment images at settlement (#37141)
This commit is contained in:
parent
60c7f847c1
commit
2ad6c42143
12 changed files with 262 additions and 47 deletions
7
packages/core/test/lib/image.ts
Normal file
7
packages/core/test/lib/image.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import { Image } from "@opencode-ai/core/image"
|
||||
import { Effect, Layer } from "effect"
|
||||
|
||||
/** Passthrough resizer for tests that build ToolRegistry.node without a Location. */
|
||||
export const imagePassthrough = Layer.mock(Image.Service, {
|
||||
normalize: (_resource, content) => Effect.succeed(content),
|
||||
})
|
||||
|
|
@ -29,7 +29,9 @@ import { McpTool } from "@opencode-ai/core/tool/mcp"
|
|||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { Deferred, Effect, Exit, Fiber, Layer, Schema, Stream } from "effect"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { location } from "./fixture/location"
|
||||
import { settleTool, toolDefinitions, toolIdentity, waitForTool } from "./lib/tool"
|
||||
|
||||
|
|
@ -250,6 +252,7 @@ const it = testEffect(
|
|||
[PermissionV2.node, permissions],
|
||||
[EventV2.node, events],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
]),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { DateTime, Effect, Fiber, Layer, Schema, Stream } from "effect"
|
||||
import { DateTime, Effect, Fiber, Layer, LayerMap, Schema, Stream } from "effect"
|
||||
import { mkdtemp, rm } from "fs/promises"
|
||||
import { tmpdir } from "os"
|
||||
import path from "path"
|
||||
|
|
@ -24,7 +24,10 @@ import { SessionExecution } from "@opencode-ai/core/session/execution"
|
|||
import { SessionPending } from "@opencode-ai/core/session/pending"
|
||||
import { SessionPendingTable, SessionMessageTable, SessionTable } from "@opencode-ai/core/session/sql"
|
||||
import { SessionStore } from "@opencode-ai/core/session/store"
|
||||
import { LocationServiceMap } from "@opencode-ai/core/location-service-map"
|
||||
import type { LocationServices } from "@opencode-ai/core/location-services"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
|
||||
const executionCalls: SessionV2.ID[] = []
|
||||
const interruptCalls: SessionV2.ID[] = []
|
||||
|
|
@ -49,10 +52,22 @@ const execution = Layer.succeed(
|
|||
awaitIdle: () => Effect.void,
|
||||
}),
|
||||
)
|
||||
const locations = Layer.effect(
|
||||
LocationServiceMap.Service,
|
||||
LayerMap.make(
|
||||
() =>
|
||||
// Attachment admission only needs the location-scoped Image service.
|
||||
// oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion
|
||||
imagePassthrough as unknown as Layer.Layer<LocationServices>,
|
||||
),
|
||||
)
|
||||
const it = testEffect(
|
||||
AppNodeBuilder.build(
|
||||
LayerNode.group([Database.node, EventV2.node, SessionProjector.node, SessionStore.node, SessionV2.node]),
|
||||
[[SessionExecution.node, execution]],
|
||||
[
|
||||
[SessionExecution.node, execution],
|
||||
[LocationServiceMap.node, locations],
|
||||
],
|
||||
),
|
||||
)
|
||||
const sessionID = SessionV2.ID.make("ses_prompt_test")
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { Tool } from "@opencode-ai/core/tool/tool"
|
|||
import { AgentV2 } from "@opencode-ai/core/agent"
|
||||
import type { PermissionV2 } from "@opencode-ai/core/permission"
|
||||
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { SessionV2 } from "@opencode-ai/core/session"
|
||||
import { SessionMessage } from "@opencode-ai/core/session/message"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
|
|
@ -28,7 +29,28 @@ const outputStore = Layer.mock(ToolOutputStore.Service, {
|
|||
)
|
||||
},
|
||||
})
|
||||
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [[ToolOutputStore.node, outputStore]])
|
||||
const imageStore = Layer.mock(Image.Service, {
|
||||
normalize: (resource, content) => {
|
||||
if (resource === "corrupt.png") return Effect.fail(new Image.DecodeError({ resource }))
|
||||
if (resource === "too-large.png")
|
||||
return Effect.fail(
|
||||
new Image.SizeError({
|
||||
resource,
|
||||
width: 9_000,
|
||||
height: 9_000,
|
||||
bytes: content.content.length,
|
||||
maxWidth: 2_000,
|
||||
maxHeight: 2_000,
|
||||
maxBytes: 5,
|
||||
}),
|
||||
)
|
||||
return Effect.succeed({ ...content, content: "bm9ybWFsaXplZA==", mime: "image/jpeg" })
|
||||
},
|
||||
})
|
||||
const registryLayer = AppNodeBuilder.build(ToolRegistry.node, [
|
||||
[ToolOutputStore.node, outputStore],
|
||||
[Image.node, imageStore],
|
||||
])
|
||||
const it = testEffect(registryLayer)
|
||||
const identity = {
|
||||
agent: AgentV2.ID.make("build"),
|
||||
|
|
@ -255,6 +277,75 @@ describe("ToolRegistry", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("normalizes image tool output at settlement and drops unresizable images", () =>
|
||||
Effect.gen(function* () {
|
||||
const service = yield* ToolRegistry.Service
|
||||
yield* service.register({
|
||||
snapshot: Tool.make({
|
||||
description: "Return images",
|
||||
input: Schema.Struct({ text: Schema.String }),
|
||||
output: Schema.Struct({ text: Schema.String }),
|
||||
execute: ({ text }) => Effect.succeed({ text }),
|
||||
toModelOutput: ({ output }) => [
|
||||
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" },
|
||||
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" },
|
||||
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "corrupt.png" },
|
||||
{ type: "text", text: output.text },
|
||||
],
|
||||
}),
|
||||
}, { codemode: false })
|
||||
|
||||
const settlement = yield* settleTool(service, call("snapshot"))
|
||||
expect(settlement.output?.content).toEqual([
|
||||
{ type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" },
|
||||
{ type: "text", text: "snapshot" },
|
||||
{ type: "text", text: "[1 image omitted: could not be decoded.]" },
|
||||
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("normalizes image progress content before it is published", () =>
|
||||
Effect.gen(function* () {
|
||||
const service = yield* ToolRegistry.Service
|
||||
yield* service.register({
|
||||
progressive: Tool.make({
|
||||
description: "Emit image progress",
|
||||
input: Schema.Struct({ text: Schema.String }),
|
||||
output: Schema.Struct({ text: Schema.String }),
|
||||
execute: ({ text }, context) =>
|
||||
context
|
||||
.progress({
|
||||
structured: { stage: "capture" },
|
||||
content: [
|
||||
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "frame.png" },
|
||||
{ type: "file", data: "aW1hZ2U=", mime: "image/png", name: "too-large.png" },
|
||||
],
|
||||
})
|
||||
.pipe(Effect.as({ text })),
|
||||
}),
|
||||
}, { codemode: false })
|
||||
|
||||
const updates: ToolRegistry.Progress[] = []
|
||||
yield* settleTool(service, {
|
||||
...call("progressive"),
|
||||
progress: (update) =>
|
||||
Effect.sync(() => {
|
||||
updates.push(update)
|
||||
}),
|
||||
})
|
||||
expect(updates).toEqual([
|
||||
{
|
||||
structured: { stage: "capture" },
|
||||
content: [
|
||||
{ type: "file", uri: "data:image/jpeg;base64,bm9ybWFsaXplZA==", mime: "image/jpeg", name: "frame.png" },
|
||||
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||
],
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("enforces transformed codecs at execution and projection boundaries", () =>
|
||||
Effect.gen(function* () {
|
||||
const service = yield* ToolRegistry.Service
|
||||
|
|
|
|||
|
|
@ -8,7 +8,9 @@ import { SessionV2 } from "@opencode-ai/core/session"
|
|||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { QuestionTool } from "@opencode-ai/core/tool/question"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
||||
|
|
@ -84,6 +86,7 @@ const it = testEffect(
|
|||
[PermissionV2.node, permission],
|
||||
[Form.node, form],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
]),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -291,6 +291,9 @@ describe("ReadTool", () => {
|
|||
name: "pixel.png",
|
||||
mime: "image/png",
|
||||
encoding: "base64",
|
||||
// Image base64 is carried by the content file item only; structured is slimmed
|
||||
// so the original bytes are never persisted twice.
|
||||
content: "",
|
||||
})
|
||||
expect(settled.output?.content).toMatchObject([
|
||||
{ type: "text", text: "Image read successfully" },
|
||||
|
|
@ -364,7 +367,7 @@ describe("ReadTool", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects invalid image data returned by the filesystem", () =>
|
||||
it.effect("drops undecodable image data at settlement", () =>
|
||||
Effect.gen(function* () {
|
||||
readResult = {
|
||||
uri: "file:///truncated.png",
|
||||
|
|
@ -381,11 +384,17 @@ describe("ReadTool", () => {
|
|||
...toolIdentity,
|
||||
call: { type: "tool-call", id: "call-truncated-image", name: "read", input: { path: "truncated.png" } },
|
||||
}),
|
||||
).toEqual({ type: "error", value: "Image could not be decoded: truncated.png" })
|
||||
).toEqual({
|
||||
type: "content",
|
||||
value: [
|
||||
{ type: "text", text: "Image read successfully" },
|
||||
{ type: "text", text: "[1 image omitted: could not be decoded.]" },
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("rejects oversized images when resizing is disabled", () =>
|
||||
it.effect("drops oversized images at settlement when resizing is disabled", () =>
|
||||
Effect.gen(function* () {
|
||||
const photon = yield* Effect.promise(() => import("@silvia-odwyer/photon-node"))
|
||||
const source = new photon.PhotonImage(new Uint8Array(Array.from({ length: 16 * 4 }, () => 255)), 16, 1)
|
||||
|
|
@ -409,14 +418,20 @@ describe("ReadTool", () => {
|
|||
}),
|
||||
]
|
||||
const registry = yield* ToolRegistry.Service
|
||||
const result = yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: { type: "tool-call", id: "call-wide-image", name: "read", input: { path: "wide.png" } },
|
||||
})
|
||||
|
||||
expect(result.type).toBe("error")
|
||||
if (result.type === "error") expect(result.value).toContain("exceeding configured limits 4x2000")
|
||||
expect(
|
||||
yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: { type: "tool-call", id: "call-wide-image", name: "read", input: { path: "wide.png" } },
|
||||
}),
|
||||
).toEqual({
|
||||
type: "content",
|
||||
value: [
|
||||
{ type: "text", text: "Image read successfully" },
|
||||
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
@ -460,7 +475,7 @@ describe("ReadTool", () => {
|
|||
}),
|
||||
)
|
||||
|
||||
it.effect("enforces max base64 bytes after resize attempts", () =>
|
||||
it.effect("drops images that cannot fit max base64 bytes after resize attempts", () =>
|
||||
Effect.gen(function* () {
|
||||
const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
|
||||
readResult = {
|
||||
|
|
@ -481,14 +496,20 @@ describe("ReadTool", () => {
|
|||
}),
|
||||
]
|
||||
const registry = yield* ToolRegistry.Service
|
||||
const result = yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: { type: "tool-call", id: "call-max-bytes", name: "read", input: { path: "pixel.png" } },
|
||||
})
|
||||
|
||||
expect(result.type).toBe("error")
|
||||
if (result.type === "error") expect(result.value).toContain("/1 bytes")
|
||||
expect(
|
||||
yield* executeTool(registry, {
|
||||
sessionID,
|
||||
...toolIdentity,
|
||||
call: { type: "tool-call", id: "call-max-bytes", name: "read", input: { path: "pixel.png" } },
|
||||
}),
|
||||
).toEqual({
|
||||
type: "content",
|
||||
value: [
|
||||
{ type: "text", text: "Image read successfully" },
|
||||
{ type: "text", text: "[1 image omitted: could not be resized below the image size limit.]" },
|
||||
],
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ import { SkillTool } from "@opencode-ai/core/tool/skill"
|
|||
import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { tmpdir } from "./fixture/tmpdir"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { it } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { FSUtil } from "@opencode-ai/core/fs-util"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
|
@ -90,6 +92,7 @@ describe("SkillTool", () => {
|
|||
[PermissionV2.node, permission],
|
||||
[SkillV2.node, skills],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
|||
import { WebFetchTool } from "@opencode-ai/core/tool/webfetch"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
||||
const webFetchToolNode = makeLocationNode({
|
||||
|
|
@ -50,6 +52,7 @@ const toolLayer = (replacements: LayerNode.Replacements = []) =>
|
|||
AppNodeBuilder.build(LayerNode.group([ToolRegistry.node, ToolRegistry.toolsNode, webFetchToolNode]), [
|
||||
[PermissionV2.node, permission],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
...replacements,
|
||||
])
|
||||
const it = testEffect(toolLayer([[LayerNodePlatform.httpClient, http]]))
|
||||
|
|
|
|||
|
|
@ -10,7 +10,9 @@ import { ToolRegistry } from "@opencode-ai/core/tool/registry"
|
|||
import { WebSearchTool } from "@opencode-ai/core/tool/websearch"
|
||||
import { ToolOutputStore } from "@opencode-ai/core/tool-output-store"
|
||||
import { makeLocationNode } from "@opencode-ai/core/effect/app-node"
|
||||
import { Image } from "@opencode-ai/core/image"
|
||||
import { testEffect } from "./lib/effect"
|
||||
import { imagePassthrough } from "./lib/image"
|
||||
import { toolIdentity, executeTool, registerToolPlugin, settleTool, toolDefinitions } from "./lib/tool"
|
||||
|
||||
const webSearchToolNode = makeLocationNode({
|
||||
|
|
@ -138,6 +140,7 @@ const it = testEffect(
|
|||
[LayerNodePlatform.httpClient, http],
|
||||
[WebSearchTool.configNode, websearchConfig],
|
||||
[ToolOutputStore.node, ToolOutputStore.nodeWithoutConfig],
|
||||
[Image.node, imagePassthrough],
|
||||
],
|
||||
),
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue