refactor(core): unify v2 tool architecture (#31168)
This commit is contained in:
Kit Langton
GitHub
parent
effd27b239
commit
660a00d317
46 changed files with 2297 additions and 2041 deletions
|
|
@ -1,12 +1,13 @@
|
|||
export * as WebFetchTool from "./webfetch"
|
||||
|
||||
import { Tool, ToolFailure, toolText } from "@opencode-ai/llm"
|
||||
import { Cause, Duration, Effect, Layer, Schema, Stream } from "effect"
|
||||
import { ToolFailure, toolText } from "@opencode-ai/llm"
|
||||
import { Duration, Effect, Layer, Schema, Stream } from "effect"
|
||||
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
|
||||
import { Parser } from "htmlparser2"
|
||||
import TurndownService from "turndown"
|
||||
import { ToolOutputStore } from "../tool-output-store"
|
||||
import { ToolRegistry } from "./registry"
|
||||
import { PermissionV2 } from "../permission"
|
||||
import { Tool } from "./tool"
|
||||
import { Tools } from "./tools"
|
||||
|
||||
export const name = "webfetch"
|
||||
export const MAX_RESPONSE_BYTES = 5 * 1024 * 1024
|
||||
|
|
@ -34,8 +35,6 @@ const Success = Schema.Struct({
|
|||
contentType: Schema.String,
|
||||
format: Parameters.fields.format,
|
||||
output: Schema.String,
|
||||
truncated: Schema.Boolean,
|
||||
outputPath: Schema.String.pipe(Schema.optional),
|
||||
})
|
||||
|
||||
type Format = (typeof Parameters.Type)["format"]
|
||||
|
|
@ -49,6 +48,7 @@ const acceptHeader = (format: Format) => {
|
|||
case "html":
|
||||
return "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1"
|
||||
}
|
||||
return "*/*"
|
||||
}
|
||||
|
||||
const headers = (format: Format, userAgent: string) => ({
|
||||
|
|
@ -89,15 +89,17 @@ const collectBody = (response: HttpClientResponse.HttpClientResponse) =>
|
|||
Effect.gen(function* () {
|
||||
const contentLength = response.headers["content-length"]
|
||||
if (contentLength && Number.parseInt(contentLength, 10) > MAX_RESPONSE_BYTES) {
|
||||
return yield* Effect.die(new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`))
|
||||
return yield* Effect.fail(new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`))
|
||||
}
|
||||
const chunks: Uint8Array[] = []
|
||||
let size = 0
|
||||
yield* Stream.runForEach(response.stream, (chunk) =>
|
||||
Effect.sync(() => {
|
||||
Effect.gen(function* () {
|
||||
size += chunk.byteLength
|
||||
if (size > MAX_RESPONSE_BYTES) throw new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`)
|
||||
if (size > MAX_RESPONSE_BYTES)
|
||||
return yield* Effect.fail(new Error(`Response too large (exceeds ${MAX_RESPONSE_BYTES} byte limit)`))
|
||||
chunks.push(chunk)
|
||||
return undefined
|
||||
}),
|
||||
)
|
||||
return Buffer.concat(chunks, size)
|
||||
|
|
@ -115,9 +117,6 @@ const isTextualMime = (mime: string) =>
|
|||
mime.endsWith("+xml") ||
|
||||
mime === "application/javascript" ||
|
||||
mime === "application/x-javascript"
|
||||
const outputMime = (format: Format) =>
|
||||
format === "markdown" ? "text/markdown" : format === "html" ? "text/html" : "text/plain"
|
||||
|
||||
const convert = (content: string, contentType: string, format: Format) => {
|
||||
if (!contentType.includes("text/html")) return content
|
||||
if (format === "markdown") return convertHTMLToMarkdown(content)
|
||||
|
|
@ -125,71 +124,64 @@ const convert = (content: string, contentType: string, format: Format) => {
|
|||
return content
|
||||
}
|
||||
|
||||
const definition = Tool.make({
|
||||
description,
|
||||
parameters: Parameters,
|
||||
success: Success,
|
||||
toModelOutput: ({ output }) => [toolText({ type: "text", text: output.output })],
|
||||
})
|
||||
|
||||
export const layer = Layer.effectDiscard(
|
||||
Effect.gen(function* () {
|
||||
const registry = yield* ToolRegistry.Service
|
||||
const tools = yield* Tools.Service
|
||||
const http = yield* HttpClient.HttpClient
|
||||
const resources = yield* ToolOutputStore.Service
|
||||
const permission = yield* PermissionV2.Service
|
||||
|
||||
yield* registry.contribute((editor) =>
|
||||
editor.set(name, {
|
||||
tool: definition,
|
||||
outputPaths: (output) => (output.outputPath ? [output.outputPath] : []),
|
||||
execute: ({ parameters, sessionID, call, assertPermission }) =>
|
||||
Effect.gen(function* () {
|
||||
const parsed = new URL(parameters.url)
|
||||
assertHttpUrl(parsed)
|
||||
yield* tools
|
||||
.register({
|
||||
[name]: Tool.make({
|
||||
description,
|
||||
input: Parameters,
|
||||
output: Success,
|
||||
toModelOutput: ({ output }) => [toolText({ type: "text", text: output.output })],
|
||||
execute: (input, context) =>
|
||||
Effect.gen(function* () {
|
||||
yield* Effect.try({
|
||||
try: () => assertHttpUrl(new URL(input.url)),
|
||||
catch: (error) => error,
|
||||
})
|
||||
|
||||
yield* assertPermission({ action: name, resources: [parameters.url], save: ["*"], metadata: parameters })
|
||||
yield* permission.assert({
|
||||
action: name,
|
||||
resources: [input.url],
|
||||
save: ["*"],
|
||||
metadata: input,
|
||||
sessionID: context.sessionID,
|
||||
agent: context.agent,
|
||||
source: { type: "tool", messageID: context.assistantMessageID, callID: context.toolCallID },
|
||||
})
|
||||
|
||||
const { body, contentType } = yield* Effect.gen(function* () {
|
||||
const response = yield* execute(http, parameters.url, parameters.format).pipe(
|
||||
Effect.catchIf(isCloudflareChallenge, () =>
|
||||
execute(http, parameters.url, parameters.format, "opencode"),
|
||||
),
|
||||
const { body, contentType } = yield* Effect.gen(function* () {
|
||||
const response = yield* execute(http, input.url, input.format).pipe(
|
||||
Effect.catchIf(isCloudflareChallenge, () => execute(http, input.url, input.format, "opencode")),
|
||||
)
|
||||
const contentType = response.headers["content-type"] || ""
|
||||
const mime = mimeFrom(contentType)
|
||||
if (isImageAttachment(mime))
|
||||
return yield* Effect.fail(new Error(`Unsupported fetched image content type: ${mime}`))
|
||||
if (!isTextualMime(mime))
|
||||
return yield* Effect.fail(new Error(`Unsupported fetched file content type: ${mime}`))
|
||||
return { body: yield* collectBody(response), contentType }
|
||||
}).pipe(
|
||||
Effect.timeoutOrElse({
|
||||
duration: Duration.seconds(input.timeout ?? DEFAULT_TIMEOUT_SECONDS),
|
||||
orElse: () => Effect.fail(new Error("Request timed out")),
|
||||
}),
|
||||
)
|
||||
const contentType = response.headers["content-type"] || ""
|
||||
const mime = mimeFrom(contentType)
|
||||
if (isImageAttachment(mime)) throw new Error(`Unsupported fetched image content type: ${mime}`)
|
||||
if (!isTextualMime(mime)) throw new Error(`Unsupported fetched file content type: ${mime}`)
|
||||
return { body: yield* collectBody(response), contentType }
|
||||
}).pipe(
|
||||
Effect.timeoutOrElse({
|
||||
duration: Duration.seconds(parameters.timeout ?? DEFAULT_TIMEOUT_SECONDS),
|
||||
orElse: () => Effect.die(new Error("Request timed out")),
|
||||
}),
|
||||
)
|
||||
const content = convert(new TextDecoder().decode(body), contentType, parameters.format)
|
||||
const truncated = yield* resources.truncate({
|
||||
sessionID,
|
||||
toolCallID: call.id,
|
||||
content,
|
||||
mime: outputMime(parameters.format),
|
||||
})
|
||||
return {
|
||||
url: parameters.url,
|
||||
contentType,
|
||||
format: parameters.format,
|
||||
output: truncated.content,
|
||||
truncated: truncated.truncated,
|
||||
...(truncated.truncated ? { outputPath: truncated.outputPath } : {}),
|
||||
}
|
||||
}).pipe(
|
||||
Effect.catchCause((cause) =>
|
||||
Effect.fail(
|
||||
new ToolFailure({ message: `Unable to fetch ${parameters.url}`, error: Cause.squash(cause) }),
|
||||
),
|
||||
),
|
||||
),
|
||||
}),
|
||||
)
|
||||
const content = convert(new TextDecoder().decode(body), contentType, input.format)
|
||||
return {
|
||||
url: input.url,
|
||||
contentType,
|
||||
format: input.format,
|
||||
output: content,
|
||||
}
|
||||
}).pipe(Effect.mapError(() => new ToolFailure({ message: `Unable to fetch ${input.url}` }))),
|
||||
}),
|
||||
})
|
||||
.pipe(Effect.orDie)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue