feat(core): normalize tool and attachment images at settlement (#37141)
This commit is contained in:
Aiden Cline
GitHub
parent
60c7f847c1
commit
2ad6c42143
12 changed files with 262 additions and 47 deletions
|
|
@ -37,6 +37,7 @@ import { Snapshot } from "./snapshot"
|
|||
import { SessionRevert } from "./session/revert"
|
||||
import { Session } from "@opencode-ai/schema/session"
|
||||
import { FSUtil } from "./fs-util"
|
||||
import { Image } from "./image"
|
||||
import { Mime } from "./mime"
|
||||
import type { EventLog } from "@opencode-ai/schema/event-log"
|
||||
import { SkillV2 } from "./skill"
|
||||
|
|
@ -531,9 +532,13 @@ const layer = Layer.effect(
|
|||
// continues from the reverted boundary rather than stale post-boundary history.
|
||||
if (session.revert)
|
||||
yield* SessionRevert.commit(session).pipe(Effect.provideService(EventV2.Service, events))
|
||||
const prompt = yield* resolvePrompt({ text: input.text, files: input.files, agents: input.agents }).pipe(
|
||||
Effect.provideService(FSUtil.Service, fs),
|
||||
)
|
||||
// Resolved lazily so prompt admission only boots location services when an
|
||||
// image attachment actually needs the resizer.
|
||||
const image = Image.Service.pipe(Effect.provide(locations.get(session.location)))
|
||||
const prompt = yield* resolvePrompt(
|
||||
{ text: input.text, files: input.files, agents: input.agents },
|
||||
image,
|
||||
).pipe(Effect.provideService(FSUtil.Service, fs))
|
||||
const messageID = input.id ?? SessionMessage.ID.create()
|
||||
const admittedInput = SessionPending.Message.make({
|
||||
type: "user",
|
||||
|
|
@ -859,10 +864,13 @@ function synthesizeTerminalShellInfo(started: ShellSchema.Info): ShellSchema.Inf
|
|||
}
|
||||
}
|
||||
|
||||
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (input: PromptInput.Prompt) {
|
||||
const resolvePrompt = Effect.fn("V2Session.resolvePrompt")(function* (
|
||||
input: PromptInput.Prompt,
|
||||
image: Effect.Effect<Image.Interface>,
|
||||
) {
|
||||
const fs = yield* FSUtil.Service
|
||||
const files = input.files
|
||||
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file), { concurrency: 8 })
|
||||
? yield* Effect.forEach(input.files, (file) => materializeAttachment(fs, file, image), { concurrency: 8 })
|
||||
: undefined
|
||||
return Prompt.make({ text: input.text, agents: input.agents, files })
|
||||
})
|
||||
|
|
@ -872,6 +880,7 @@ const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024
|
|||
const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(function* (
|
||||
fs: FSUtil.Interface,
|
||||
input: PromptInput.FileAttachment,
|
||||
image: Effect.Effect<Image.Interface>,
|
||||
) {
|
||||
const resolved = input.uri.startsWith("data:")
|
||||
? {
|
||||
|
|
@ -900,9 +909,15 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
|
|||
.join("\n"),
|
||||
)
|
||||
: resolved.bytes
|
||||
return FileAttachment.create({
|
||||
data: Base64.make(Buffer.from(content).toString("base64")),
|
||||
const normalized = yield* normalizeImageAttachment(
|
||||
input,
|
||||
Base64.make(Buffer.from(content).toString("base64")),
|
||||
mime,
|
||||
image,
|
||||
)
|
||||
return FileAttachment.create({
|
||||
data: normalized.data,
|
||||
mime: normalized.mime,
|
||||
source: resolved.source,
|
||||
name: input.name ?? resolved.name,
|
||||
description: input.description,
|
||||
|
|
@ -910,6 +925,23 @@ const materializeAttachment = Effect.fn("V2Session.materializeAttachment")(funct
|
|||
})
|
||||
})
|
||||
|
||||
const normalizeImageAttachment = Effect.fn("V2Session.normalizeImageAttachment")(function* (
|
||||
input: PromptInput.FileAttachment,
|
||||
data: Base64,
|
||||
mime: string,
|
||||
image: Effect.Effect<Image.Interface>,
|
||||
) {
|
||||
if (!mime.startsWith("image/")) return { data, mime }
|
||||
const service = yield* image
|
||||
const label = input.name ?? (input.uri.startsWith("data:") ? "inline attachment" : input.uri)
|
||||
const content = { uri: label, content: data, encoding: "base64" as const, mime }
|
||||
const normalized = yield* service.normalize(label, content).pipe(
|
||||
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)),
|
||||
Effect.mapError((error) => new AttachmentError({ uri: label, message: error.message })),
|
||||
)
|
||||
return { data: Base64.make(normalized.content), mime: normalized.mime }
|
||||
})
|
||||
|
||||
const readFileAttachment = Effect.fn("V2Session.readFileAttachment")(function* (fs: FSUtil.Interface, uri: string) {
|
||||
const url = yield* Effect.try({
|
||||
try: () => new URL(uri),
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import { ToolFailure } from "@opencode-ai/ai"
|
|||
import { Effect, Schema } from "effect"
|
||||
import { FileSystem } from "../filesystem"
|
||||
import { FSUtil } from "../fs-util"
|
||||
import { Image } from "../image"
|
||||
import { Location } from "../location"
|
||||
import { LocationMutation } from "../location-mutation"
|
||||
import { PermissionV2 } from "../permission"
|
||||
|
|
@ -35,7 +34,6 @@ export const Plugin = {
|
|||
effect: Effect.fn("ReadTool.Plugin")(function* (ctx: PluginContext) {
|
||||
const reader = yield* ReadToolFileSystem.Service
|
||||
const mutation = yield* LocationMutation.Service
|
||||
const image = yield* Image.Service
|
||||
const permission = yield* PermissionV2.Service
|
||||
const sessionInstructions = yield* SessionInstructions.Service
|
||||
const fs = yield* FSUtil.Service
|
||||
|
|
@ -50,6 +48,12 @@ export const Plugin = {
|
|||
"Read a text file or supported image, page through a large UTF-8 text file by line offset, or list a directory page. Relative paths resolve from the current location; absolute paths inside it are accepted, while external absolute paths require external_directory approval.",
|
||||
input: Input,
|
||||
output: Output,
|
||||
structured: Schema.toEncoded(Output),
|
||||
// Image base64 reaches the model through content items (normalized generically
|
||||
// at tool settlement); persisting a second copy in structured would store the
|
||||
// original unresized bytes in the message row.
|
||||
toStructuredOutput: ({ output }) =>
|
||||
"encoding" in output && output.encoding === "base64" ? { ...output, content: "" } : output,
|
||||
toModelOutput: ({ input, output }) => {
|
||||
if (!("encoding" in output) || output.encoding !== "base64" || !SUPPORTED_IMAGE_MIMES.has(output.mime))
|
||||
return []
|
||||
|
|
@ -117,21 +121,14 @@ export const Plugin = {
|
|||
Effect.catch(() => Effect.void),
|
||||
Effect.catchDefect(() => Effect.void),
|
||||
)
|
||||
if ("encoding" in content && content.encoding === "base64" && SUPPORTED_IMAGE_MIMES.has(content.mime)) {
|
||||
return yield* image
|
||||
.normalize(resource, { ...content, encoding: "base64" })
|
||||
.pipe(Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(content)))
|
||||
}
|
||||
if ("encoding" in content && content.encoding === "base64")
|
||||
if ("encoding" in content && content.encoding === "base64" && !SUPPORTED_IMAGE_MIMES.has(content.mime))
|
||||
return yield* Effect.fail(new ReadToolFileSystem.BinaryFileError({ resource }))
|
||||
return content
|
||||
}).pipe(
|
||||
Effect.mapError((error) => {
|
||||
const message =
|
||||
error instanceof ReadToolFileSystem.BinaryFileError ||
|
||||
error instanceof ReadToolFileSystem.MediaIngestLimitError ||
|
||||
error instanceof Image.DecodeError ||
|
||||
error instanceof Image.SizeError
|
||||
error instanceof ReadToolFileSystem.MediaIngestLimitError
|
||||
? error.message
|
||||
: `Unable to read ${input.path}`
|
||||
return new ToolFailure({ message, error })
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ export * as ToolRegistry from "./registry"
|
|||
import { ToolOutput, type ToolCall, type ToolDefinition, type ToolResultValue } from "@opencode-ai/ai"
|
||||
import { Context, Effect, Layer, Scope } from "effect"
|
||||
import type { AgentV2 } from "../agent"
|
||||
import { Image } from "../image"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { SessionMessage } from "../session/message"
|
||||
import { SessionSchema } from "../session/schema"
|
||||
|
|
@ -57,6 +58,40 @@ const registryLayer = Layer.effect(
|
|||
Effect.gen(function* () {
|
||||
const resources = yield* ToolOutputStore.Service
|
||||
const toolHooks = yield* ToolHooks.Service
|
||||
const image = yield* Image.Service
|
||||
|
||||
type NormalizedItem = ToolOutput["content"][number] | "decode" | "size"
|
||||
const normalizeImages = Effect.fn("ToolRegistry.normalizeImages")(function* (content: ToolOutput["content"]) {
|
||||
const normalized = yield* Effect.forEach(content, (item): Effect.Effect<NormalizedItem> => {
|
||||
if (item.type !== "file" || !item.mime.startsWith("image/")) return Effect.succeed(item)
|
||||
// RFC 2397 permits parameters between the mime and ";base64".
|
||||
const base64 = /^data:[^,]*;base64,(.*)$/s.exec(item.uri)?.[1]
|
||||
if (base64 === undefined) return Effect.succeed(item)
|
||||
const resource = item.name ?? `${item.mime} tool output`
|
||||
return image
|
||||
.normalize(resource, { uri: resource, content: base64, encoding: "base64", mime: item.mime })
|
||||
.pipe(
|
||||
Effect.map((result) => ({
|
||||
...item,
|
||||
uri: `data:${result.mime};base64,${result.content}`,
|
||||
mime: result.mime,
|
||||
})),
|
||||
Effect.catchTag("Image.ResizerUnavailableError", () => Effect.succeed(item)),
|
||||
Effect.catchTag("Image.DecodeError", () => Effect.succeed("decode" as const)),
|
||||
Effect.catchTag("Image.SizeError", () => Effect.succeed("size" as const)),
|
||||
)
|
||||
})
|
||||
const note = (reason: "decode" | "size", text: string) => {
|
||||
const count = normalized.filter((item) => item === reason).length
|
||||
if (count === 0) return []
|
||||
return [{ type: "text" as const, text: `[${count} image${count === 1 ? "" : "s"} omitted: ${text}]` }]
|
||||
}
|
||||
return [
|
||||
...normalized.filter((item) => typeof item !== "string"),
|
||||
...note("decode", "could not be decoded."),
|
||||
...note("size", "could not be resized below the image size limit."),
|
||||
]
|
||||
})
|
||||
type Registration = {
|
||||
readonly tool: AnyTool
|
||||
readonly name: string
|
||||
|
|
@ -84,10 +119,11 @@ const registryLayer = Layer.effect(
|
|||
agent: input.agent,
|
||||
messageID: input.messageID,
|
||||
callID: input.call.id,
|
||||
progress: (update) =>
|
||||
input.progress?.({
|
||||
structured: update.structured,
|
||||
content: (update.content ?? []).map((part) =>
|
||||
progress: (update) => {
|
||||
const progress = input.progress
|
||||
if (!progress) return Effect.void
|
||||
return normalizeImages(
|
||||
(update.content ?? []).map((part) =>
|
||||
part.type === "text"
|
||||
? { type: "text" as const, text: part.text }
|
||||
: {
|
||||
|
|
@ -97,7 +133,8 @@ const registryLayer = Layer.effect(
|
|||
name: part.name,
|
||||
},
|
||||
),
|
||||
}) ?? Effect.void,
|
||||
).pipe(Effect.flatMap((content) => progress({ structured: update.structured, content })))
|
||||
},
|
||||
},
|
||||
).pipe(
|
||||
Effect.map((output) => ({ output })),
|
||||
|
|
@ -115,7 +152,7 @@ const registryLayer = Layer.effect(
|
|||
const bounded = yield* resources.bound({
|
||||
sessionID: input.sessionID,
|
||||
callID: input.call.id,
|
||||
output: pending.output,
|
||||
output: { structured: pending.output.structured, content: yield* normalizeImages(pending.output.content) },
|
||||
})
|
||||
const result = ToolOutput.toResultValue(bounded.output)
|
||||
settlement =
|
||||
|
|
@ -232,11 +269,11 @@ function whollyDisabled(action: string, rules: PermissionV2.Ruleset) {
|
|||
export const node = makeLocationNode({
|
||||
service: Service,
|
||||
layer,
|
||||
deps: [ToolOutputStore.node, ToolHooks.node],
|
||||
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
|
||||
})
|
||||
|
||||
export const toolsNode = makeLocationNode({
|
||||
service: Tools.Service,
|
||||
layer,
|
||||
deps: [ToolOutputStore.node, ToolHooks.node],
|
||||
deps: [ToolOutputStore.node, ToolHooks.node, Image.node],
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue