fix(core): make V2 reads media-aware and binary-safe (#31038)
This commit is contained in:
Kit Langton
GitHub
parent
f750deaa3e
commit
83dca45dd5
26 changed files with 1709 additions and 120 deletions
|
|
@ -303,14 +303,17 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult
|
|||
})
|
||||
|
||||
const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: MediaPart) {
|
||||
if (!part.mediaType.startsWith("image/"))
|
||||
return yield* invalid(`Anthropic Messages user media content only supports images`)
|
||||
const media = yield* ProviderShared.validateMedia(
|
||||
"Anthropic Messages",
|
||||
part,
|
||||
new Set<string>(ProviderShared.IMAGE_MIMES),
|
||||
)
|
||||
return {
|
||||
type: "image" as const,
|
||||
source: {
|
||||
type: "base64" as const,
|
||||
media_type: part.mediaType,
|
||||
data: ProviderShared.mediaBase64(part),
|
||||
media_type: media.mime,
|
||||
data: media.base64,
|
||||
},
|
||||
} satisfies AnthropicImageBlock
|
||||
})
|
||||
|
|
@ -321,16 +324,19 @@ const lowerToolResultContentItem = Effect.fn("AnthropicMessages.lowerToolResultC
|
|||
item: ToolResultContentPart,
|
||||
) {
|
||||
if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
|
||||
if (item.mediaType.startsWith("image/"))
|
||||
return {
|
||||
type: "image" as const,
|
||||
source: {
|
||||
type: "base64" as const,
|
||||
media_type: item.mediaType,
|
||||
data: ProviderShared.mediaBase64(item),
|
||||
},
|
||||
} satisfies AnthropicImageBlock
|
||||
return yield* invalid(`Anthropic Messages tool-result media content only supports images, got ${item.mediaType}`)
|
||||
const media = yield* ProviderShared.validateMedia(
|
||||
"Anthropic Messages",
|
||||
item,
|
||||
new Set<string>(ProviderShared.IMAGE_MIMES),
|
||||
)
|
||||
return {
|
||||
type: "image" as const,
|
||||
source: {
|
||||
type: "base64" as const,
|
||||
media_type: media.mime,
|
||||
data: media.base64,
|
||||
},
|
||||
} satisfies AnthropicImageBlock
|
||||
})
|
||||
|
||||
const lowerToolResultContent = Effect.fn("AnthropicMessages.lowerToolResultContent")(function* (part: ToolResultPart) {
|
||||
|
|
|
|||
|
|
@ -14,12 +14,14 @@ import {
|
|||
type TextPart,
|
||||
type ToolCallPart,
|
||||
type ToolDefinition,
|
||||
type ToolResultContentPart,
|
||||
} from "../schema"
|
||||
import { JsonObject, optionalArray, ProviderShared } from "./shared"
|
||||
import { GeminiToolSchema } from "./utils/gemini-tool-schema"
|
||||
import { Lifecycle } from "./utils/lifecycle"
|
||||
|
||||
const ADAPTER = "gemini"
|
||||
const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
|
||||
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
|
||||
|
||||
// =============================================================================
|
||||
|
|
@ -140,8 +142,6 @@ interface ParserState {
|
|||
readonly reasoningSignature?: string
|
||||
}
|
||||
|
||||
const mediaData = ProviderShared.mediaBytes
|
||||
|
||||
// =============================================================================
|
||||
// Tool Schema Conversion
|
||||
// =============================================================================
|
||||
|
|
@ -180,8 +180,11 @@ const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
|
|||
tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
|
||||
})
|
||||
|
||||
const lowerUserPart = (part: TextPart | MediaPart) =>
|
||||
part.type === "text" ? { text: part.text } : { inlineData: { mimeType: part.mediaType, data: mediaData(part) } }
|
||||
const lowerUserPart = Effect.fn("Gemini.lowerUserPart")(function* (part: TextPart | MediaPart) {
|
||||
if (part.type === "text") return { text: part.text }
|
||||
const media = yield* ProviderShared.validateMedia("Gemini", part, IMAGE_MIMES)
|
||||
return { inlineData: { mimeType: media.mime, data: media.base64 } }
|
||||
})
|
||||
|
||||
const googleMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ google: metadata })
|
||||
|
||||
|
|
@ -215,7 +218,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
|
|||
for (const part of message.content) {
|
||||
if (!ProviderShared.supportsContent(part, ["text", "media"]))
|
||||
return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"])
|
||||
parts.push(lowerUserPart(part))
|
||||
parts.push(yield* lowerUserPart(part))
|
||||
}
|
||||
contents.push({ role: "user", parts })
|
||||
continue
|
||||
|
|
@ -247,15 +250,34 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
|
|||
for (const part of message.content) {
|
||||
if (!ProviderShared.supportsContent(part, ["tool-result"]))
|
||||
return yield* ProviderShared.unsupportedContent("Gemini", "tool", ["tool-result"])
|
||||
if (part.result.type !== "content") {
|
||||
parts.push({
|
||||
functionResponse: {
|
||||
name: part.name,
|
||||
response: {
|
||||
name: part.name,
|
||||
content: ProviderShared.toolResultText(part),
|
||||
},
|
||||
},
|
||||
})
|
||||
continue
|
||||
}
|
||||
const content: ReadonlyArray<ToolResultContentPart> = part.result.value
|
||||
const text = content.filter((item) => item.type === "text").map((item) => item.text)
|
||||
parts.push({
|
||||
functionResponse: {
|
||||
name: part.name,
|
||||
response: {
|
||||
name: part.name,
|
||||
content: ProviderShared.toolResultText(part),
|
||||
content: text.join("\n"),
|
||||
},
|
||||
},
|
||||
})
|
||||
for (const item of content) {
|
||||
if (item.type === "text") continue
|
||||
const media = yield* ProviderShared.validateMedia("Gemini", item, IMAGE_MIMES)
|
||||
parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } })
|
||||
}
|
||||
}
|
||||
contents.push({ role: "user", parts })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,10 +9,12 @@ import {
|
|||
Usage,
|
||||
type FinishReason,
|
||||
type LLMRequest,
|
||||
type MediaPart,
|
||||
type ReasoningPart,
|
||||
type TextPart,
|
||||
type ToolCallPart,
|
||||
type ToolDefinition,
|
||||
type ToolResultContentPart,
|
||||
} from "../schema"
|
||||
import { isRecord, JsonObject, optionalArray, optionalNull, ProviderShared } from "./shared"
|
||||
import { OpenAIOptions } from "./utils/openai-options"
|
||||
|
|
@ -20,6 +22,7 @@ import { Lifecycle } from "./utils/lifecycle"
|
|||
import { ToolStream } from "./utils/tool-stream"
|
||||
|
||||
const ADAPTER = "openai-chat"
|
||||
const IMAGE_MIMES = new Set<string>(ProviderShared.IMAGE_MIMES)
|
||||
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
|
||||
export const PATH = "/chat/completions"
|
||||
|
||||
|
|
@ -51,9 +54,20 @@ const OpenAIChatAssistantToolCall = Schema.Struct({
|
|||
})
|
||||
type OpenAIChatAssistantToolCall = Schema.Schema.Type<typeof OpenAIChatAssistantToolCall>
|
||||
|
||||
const OpenAIChatUserContent = Schema.Union([
|
||||
Schema.Struct({ type: Schema.Literal("text"), text: Schema.String }),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("image_url"),
|
||||
image_url: Schema.Struct({ url: Schema.String }),
|
||||
}),
|
||||
])
|
||||
|
||||
const OpenAIChatMessage = Schema.Union([
|
||||
Schema.Struct({ role: Schema.Literal("system"), content: Schema.String }),
|
||||
Schema.Struct({ role: Schema.Literal("user"), content: Schema.String }),
|
||||
Schema.Struct({
|
||||
role: Schema.Literal("user"),
|
||||
content: Schema.Union([Schema.String, Schema.Array(OpenAIChatUserContent)]),
|
||||
}),
|
||||
Schema.Struct({
|
||||
role: Schema.Literal("assistant"),
|
||||
content: Schema.NullOr(Schema.String),
|
||||
|
|
@ -186,17 +200,32 @@ const lowerToolCall = (part: ToolCallPart): OpenAIChatAssistantToolCall => ({
|
|||
},
|
||||
})
|
||||
|
||||
const lowerMedia = Effect.fn("OpenAIChat.lowerMedia")(function* (
|
||||
part: Extract<MediaPart | ToolResultContentPart, { type: "media" }>,
|
||||
) {
|
||||
const media = yield* ProviderShared.validateMedia("OpenAI Chat", part, IMAGE_MIMES)
|
||||
return { type: "image_url" as const, image_url: { url: media.dataUrl } }
|
||||
})
|
||||
|
||||
const openAICompatibleReasoningContent = (native: unknown) =>
|
||||
isRecord(native) && typeof native.reasoning_content === "string" ? native.reasoning_content : undefined
|
||||
|
||||
const lowerUserMessage = Effect.fn("OpenAIChat.lowerUserMessage")(function* (message: OpenAIChatRequestMessage) {
|
||||
const content: TextPart[] = []
|
||||
const content: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
|
||||
for (const part of message.content) {
|
||||
if (!ProviderShared.supportsContent(part, ["text"]))
|
||||
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text"])
|
||||
content.push(part)
|
||||
if (part.type === "text") {
|
||||
content.push({ type: "text", text: part.text })
|
||||
continue
|
||||
}
|
||||
if (part.type === "media") {
|
||||
content.push(yield* lowerMedia(part))
|
||||
continue
|
||||
}
|
||||
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "user", ["text", "media"])
|
||||
}
|
||||
return { role: "user" as const, content: ProviderShared.joinText(content) }
|
||||
if (content.every((part) => part.type === "text"))
|
||||
return { role: "user" as const, content: content.map((part) => part.text).join("") }
|
||||
return { role: "user" as const, content }
|
||||
})
|
||||
|
||||
const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(function* (
|
||||
|
|
@ -234,35 +263,68 @@ const lowerAssistantMessage = Effect.fn("OpenAIChat.lowerAssistantMessage")(func
|
|||
|
||||
const lowerToolMessages = Effect.fn("OpenAIChat.lowerToolMessages")(function* (message: OpenAIChatRequestMessage) {
|
||||
const messages: OpenAIChatMessage[] = []
|
||||
const images: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
|
||||
for (const part of message.content) {
|
||||
if (!ProviderShared.supportsContent(part, ["tool-result"]))
|
||||
return yield* ProviderShared.unsupportedContent("OpenAI Chat", "tool", ["tool-result"])
|
||||
messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
|
||||
if (part.result.type !== "content") {
|
||||
messages.push({ role: "tool", tool_call_id: part.id, content: ProviderShared.toolResultText(part) })
|
||||
continue
|
||||
}
|
||||
const content: ReadonlyArray<ToolResultContentPart> = part.result.value
|
||||
const text = content.filter((item) => item.type === "text").map((item) => item.text)
|
||||
messages.push({ role: "tool", tool_call_id: part.id, content: text.join("\n") })
|
||||
const media = content.filter(
|
||||
(item): item is Extract<ToolResultContentPart, { type: "media" }> => item.type === "media",
|
||||
)
|
||||
images.push(...(yield* Effect.forEach(media, lowerMedia)))
|
||||
}
|
||||
return messages
|
||||
return { messages, images }
|
||||
})
|
||||
|
||||
const lowerMessage = Effect.fn("OpenAIChat.lowerMessage")(function* (message: OpenAIChatRequestMessage) {
|
||||
if (message.role === "user") return [yield* lowerUserMessage(message)]
|
||||
if (message.role === "assistant") return [yield* lowerAssistantMessage(message)]
|
||||
return yield* lowerToolMessages(message)
|
||||
return (yield* lowerToolMessages(message)).messages
|
||||
})
|
||||
|
||||
const lowerMessages = Effect.fn("OpenAIChat.lowerMessages")(function* (request: LLMRequest) {
|
||||
const system: OpenAIChatMessage[] =
|
||||
request.system.length === 0 ? [] : [{ role: "system", content: ProviderShared.joinText(request.system) }]
|
||||
const messages = [...system]
|
||||
const pendingImages: Array<Schema.Schema.Type<typeof OpenAIChatUserContent>> = []
|
||||
const flushImages = () => {
|
||||
if (pendingImages.length === 0) return
|
||||
messages.push({ role: "user", content: pendingImages.splice(0) })
|
||||
}
|
||||
for (const message of request.messages) {
|
||||
if (message.role === "system") {
|
||||
const part = yield* ProviderShared.wrappedSystemUpdate("OpenAI Chat", message)
|
||||
if (pendingImages.length > 0) {
|
||||
messages.push({ role: "user", content: [...pendingImages.splice(0), { type: "text", text: part.text }] })
|
||||
continue
|
||||
}
|
||||
const previous = messages.at(-1)
|
||||
if (previous?.role === "user")
|
||||
if (previous?.role === "user" && typeof previous.content === "string")
|
||||
messages[messages.length - 1] = { role: "user", content: `${previous.content}\n${part.text}` }
|
||||
else if (previous?.role === "user" && Array.isArray(previous.content))
|
||||
messages[messages.length - 1] = {
|
||||
role: "user",
|
||||
content: [...previous.content, { type: "text", text: part.text }],
|
||||
}
|
||||
else messages.push({ role: "user", content: part.text })
|
||||
continue
|
||||
}
|
||||
if (message.role === "tool") {
|
||||
const lowered = yield* lowerToolMessages(message)
|
||||
messages.push(...lowered.messages)
|
||||
pendingImages.push(...lowered.images)
|
||||
continue
|
||||
}
|
||||
flushImages()
|
||||
messages.push(...(yield* lowerMessage(message)))
|
||||
}
|
||||
flushImages()
|
||||
return messages
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -304,10 +304,14 @@ const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function*
|
|||
part: LLMRequest["messages"][number]["content"][number],
|
||||
) {
|
||||
if (part.type === "text") return { type: "input_text" as const, text: part.text }
|
||||
if (part.type === "media" && part.mediaType.startsWith("image/")) {
|
||||
return { type: "input_image" as const, image_url: ProviderShared.mediaDataUrl(part) }
|
||||
if (part.type === "media") {
|
||||
const media = yield* ProviderShared.validateMedia(
|
||||
"OpenAI Responses",
|
||||
part,
|
||||
new Set<string>(ProviderShared.IMAGE_MIMES),
|
||||
)
|
||||
return { type: "input_image" as const, image_url: media.dataUrl }
|
||||
}
|
||||
if (part.type === "media") return yield* invalid("OpenAI Responses user media content only supports images")
|
||||
return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"])
|
||||
})
|
||||
|
||||
|
|
@ -317,12 +321,12 @@ const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultCon
|
|||
item: ToolResultContentPart,
|
||||
) {
|
||||
if (item.type === "text") return { type: "input_text" as const, text: item.text }
|
||||
if (item.mediaType.startsWith("image/"))
|
||||
return {
|
||||
type: "input_image" as const,
|
||||
image_url: ProviderShared.mediaDataUrl(item),
|
||||
}
|
||||
return yield* invalid(`OpenAI Responses tool-result media content only supports images, got ${item.mediaType}`)
|
||||
const media = yield* ProviderShared.validateMedia(
|
||||
"OpenAI Responses",
|
||||
item,
|
||||
new Set<string>(ProviderShared.IMAGE_MIMES),
|
||||
)
|
||||
return { type: "input_image" as const, image_url: media.dataUrl }
|
||||
})
|
||||
|
||||
const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")(function* (part: ToolResultPart) {
|
||||
|
|
|
|||
|
|
@ -186,24 +186,52 @@ export const wrappedSystemUpdate = Effect.fn("ProviderShared.wrappedSystemUpdate
|
|||
export const parseToolInput = (route: string, name: string, raw: string) =>
|
||||
parseJson(route, raw || "{}", `Invalid JSON input for ${route} tool call ${name}`)
|
||||
|
||||
/**
|
||||
* Encode a `MediaPart`'s raw bytes for inclusion in a JSON request body.
|
||||
* `data: string` is assumed to already be base64 (matches caller convention
|
||||
* across Gemini / Bedrock); `data: Uint8Array` is base64-encoded here. Used
|
||||
* by every route that supports image / document inputs.
|
||||
*/
|
||||
export const mediaBytes = (part: MediaPart) =>
|
||||
typeof part.data === "string" ? part.data : Buffer.from(part.data).toString("base64")
|
||||
export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const
|
||||
export const MAX_MEDIA_ENCODED_BYTES = 8 * 1024 * 1024
|
||||
export const MAX_MEDIA_DECODED_BYTES = 6 * 1024 * 1024
|
||||
|
||||
export const mediaBase64 = (part: MediaPart) => {
|
||||
if (typeof part.data !== "string" || !part.data.startsWith("data:")) return mediaBytes(part)
|
||||
return part.data.slice(part.data.indexOf(",") + 1)
|
||||
const base64Pattern = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
|
||||
|
||||
export interface ValidatedMedia {
|
||||
readonly mime: string
|
||||
readonly base64: string
|
||||
readonly dataUrl: string
|
||||
readonly bytes: Uint8Array
|
||||
}
|
||||
|
||||
export const mediaDataUrl = (part: MediaPart) =>
|
||||
typeof part.data === "string" && part.data.startsWith("data:")
|
||||
? part.data
|
||||
: `data:${part.mediaType};base64,${mediaBytes(part)}`
|
||||
export const validateMedia = Effect.fn("ProviderShared.validateMedia")(function* (
|
||||
route: string,
|
||||
part: MediaPart,
|
||||
supportedMimes: ReadonlySet<string>,
|
||||
) {
|
||||
const mime = part.mediaType.toLowerCase()
|
||||
if (!supportedMimes.has(mime)) return yield* invalidRequest(`${route} does not support media type ${part.mediaType}`)
|
||||
|
||||
let base64: string
|
||||
if (typeof part.data !== "string") {
|
||||
if (part.data.byteLength > MAX_MEDIA_DECODED_BYTES)
|
||||
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
|
||||
base64 = Buffer.from(part.data).toString("base64")
|
||||
} else if (part.data.startsWith("data:")) {
|
||||
const match = /^data:([^;,]+);base64,([A-Za-z0-9+/]*={0,2})$/s.exec(part.data)
|
||||
if (!match) return yield* invalidRequest(`${route} media data URL must contain valid base64`)
|
||||
if (match[1]!.toLowerCase() !== mime)
|
||||
return yield* invalidRequest(`${route} media type ${part.mediaType} does not match data URL type ${match[1]}`)
|
||||
base64 = match[2]!
|
||||
} else {
|
||||
base64 = part.data
|
||||
}
|
||||
|
||||
if (Buffer.byteLength(base64, "utf8") > MAX_MEDIA_ENCODED_BYTES)
|
||||
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_ENCODED_BYTES} byte encoded limit`)
|
||||
if (!base64 || base64.length % 4 !== 0 || !base64Pattern.test(base64))
|
||||
return yield* invalidRequest(`${route} media must contain valid base64`)
|
||||
const bytes = Buffer.from(base64, "base64")
|
||||
if (bytes.byteLength > MAX_MEDIA_DECODED_BYTES)
|
||||
return yield* invalidRequest(`${route} media exceeds the ${MAX_MEDIA_DECODED_BYTES} byte decoded limit`)
|
||||
if (bytes.toString("base64") !== base64) return yield* invalidRequest(`${route} media must contain canonical base64`)
|
||||
return { mime, base64, dataUrl: `data:${mime};base64,${base64}`, bytes } satisfies ValidatedMedia
|
||||
})
|
||||
|
||||
export const trimBaseUrl = (value: string) => value.replace(/\/+$/, "")
|
||||
|
||||
|
|
|
|||
|
|
@ -49,15 +49,11 @@ const DOCUMENT_FORMATS = {
|
|||
"text/markdown": "md",
|
||||
} as const satisfies Record<string, DocumentFormat>
|
||||
|
||||
const imageBlock = (part: MediaPart, format: ImageFormat): ImageBlock => ({
|
||||
image: { format, source: { bytes: ProviderShared.mediaBytes(part) } },
|
||||
})
|
||||
|
||||
const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock => ({
|
||||
const documentBlock = (part: MediaPart, format: DocumentFormat, bytes: string): DocumentBlock => ({
|
||||
document: {
|
||||
format,
|
||||
name: part.filename ?? `document.${format}`,
|
||||
source: { bytes: ProviderShared.mediaBytes(part) },
|
||||
source: { bytes },
|
||||
},
|
||||
})
|
||||
|
||||
|
|
@ -66,15 +62,29 @@ const documentBlock = (part: MediaPart, format: DocumentFormat): DocumentBlock =
|
|||
// document block. Image MIME types not in `IMAGE_FORMATS` (e.g. `image/svg+xml`)
|
||||
// get an image-specific error so the caller knows it's a format-support issue,
|
||||
// not a kind-detection issue.
|
||||
export const lower = (part: MediaPart) => {
|
||||
export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart) {
|
||||
const mime = part.mediaType.toLowerCase()
|
||||
const imageFormat = IMAGE_FORMATS[mime as keyof typeof IMAGE_FORMATS]
|
||||
if (imageFormat) return Effect.succeed(imageBlock(part, imageFormat))
|
||||
if (imageFormat) {
|
||||
const media = yield* ProviderShared.validateMedia(
|
||||
"Bedrock Converse",
|
||||
part,
|
||||
new Set<string>(Object.keys(IMAGE_FORMATS)),
|
||||
)
|
||||
return { image: { format: imageFormat, source: { bytes: media.base64 } } } satisfies ImageBlock
|
||||
}
|
||||
if (mime.startsWith("image/"))
|
||||
return ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
|
||||
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
|
||||
const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
|
||||
if (documentFormat) return Effect.succeed(documentBlock(part, documentFormat))
|
||||
return ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
|
||||
}
|
||||
if (documentFormat) {
|
||||
const media = yield* ProviderShared.validateMedia(
|
||||
"Bedrock Converse",
|
||||
part,
|
||||
new Set<string>(Object.keys(DOCUMENT_FORMATS)),
|
||||
)
|
||||
return documentBlock(part, documentFormat, media.base64)
|
||||
}
|
||||
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
|
||||
})
|
||||
|
||||
export * as BedrockMedia from "./bedrock-media"
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ export interface Tool<Parameters extends ToolSchema<any>, Success extends ToolSc
|
|||
readonly success: Success
|
||||
readonly execute?: ToolExecute<Parameters, Success>
|
||||
readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
|
||||
readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
|
||||
/** @internal */
|
||||
readonly _decode: (input: unknown) => Effect.Effect<Schema.Schema.Type<Parameters>, Schema.SchemaError>
|
||||
/** @internal */
|
||||
|
|
@ -86,6 +87,7 @@ type TypedToolConfig = {
|
|||
readonly success: ToolSchema<any>
|
||||
readonly execute?: ToolExecute<ToolSchema<any>, ToolSchema<any>>
|
||||
readonly toModelOutput?: ToolToModelOutput<ToolSchema<any>, ToolSchema<any>>
|
||||
readonly toStructuredOutput?: (output: unknown) => unknown
|
||||
}
|
||||
|
||||
type DynamicToolConfig = {
|
||||
|
|
@ -94,6 +96,7 @@ type DynamicToolConfig = {
|
|||
readonly outputSchema?: JsonSchema.JsonSchema
|
||||
readonly execute?: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
|
||||
readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
|
||||
readonly toStructuredOutput?: (output: unknown) => unknown
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -133,6 +136,7 @@ export function make<Parameters extends ToolSchema<any>, Success extends ToolSch
|
|||
readonly success: Success
|
||||
readonly execute: ToolExecute<Parameters, Success>
|
||||
readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
|
||||
readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
|
||||
}): ExecutableTool<Parameters, Success>
|
||||
export function make<Parameters extends ToolSchema<any>, Success extends ToolSchema<any>>(config: {
|
||||
readonly description: string
|
||||
|
|
@ -140,6 +144,7 @@ export function make<Parameters extends ToolSchema<any>, Success extends ToolSch
|
|||
readonly success: Success
|
||||
readonly execute?: undefined
|
||||
readonly toModelOutput?: ToolToModelOutput<Parameters, Success>
|
||||
readonly toStructuredOutput?: (output: Success["Encoded"]) => unknown
|
||||
}): Tool<Parameters, Success>
|
||||
export function make(config: {
|
||||
readonly description: string
|
||||
|
|
@ -147,6 +152,7 @@ export function make(config: {
|
|||
readonly outputSchema?: JsonSchema.JsonSchema
|
||||
readonly execute: (params: unknown, context?: ToolExecuteContext) => Effect.Effect<unknown, ToolFailure>
|
||||
readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
|
||||
readonly toStructuredOutput?: (output: unknown) => unknown
|
||||
}): AnyExecutableTool
|
||||
export function make(config: {
|
||||
readonly description: string
|
||||
|
|
@ -154,6 +160,7 @@ export function make(config: {
|
|||
readonly outputSchema?: JsonSchema.JsonSchema
|
||||
readonly execute?: undefined
|
||||
readonly toModelOutput?: (input: ToolModelOutputInput<unknown, unknown>) => ReadonlyArray<ToolContent>
|
||||
readonly toStructuredOutput?: (output: unknown) => unknown
|
||||
}): AnyTool
|
||||
export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
|
||||
if ("jsonSchema" in config) {
|
||||
|
|
@ -163,10 +170,12 @@ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
|
|||
success: Schema.Unknown as ToolSchema<unknown>,
|
||||
execute: config.execute,
|
||||
toModelOutput: config.toModelOutput,
|
||||
toStructuredOutput: config.toStructuredOutput,
|
||||
_decode: Effect.succeed,
|
||||
_encode: Effect.succeed,
|
||||
_project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
|
||||
_legacyResult: config.toModelOutput === undefined,
|
||||
_project: (parameters, callID, output) =>
|
||||
project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
|
||||
_legacyResult: config.toModelOutput === undefined && config.toStructuredOutput === undefined,
|
||||
_definition: new ToolDefinition({
|
||||
name: "",
|
||||
description: config.description,
|
||||
|
|
@ -181,9 +190,11 @@ export function make(config: TypedToolConfig | DynamicToolConfig): AnyTool {
|
|||
success: config.success,
|
||||
execute: config.execute,
|
||||
toModelOutput: config.toModelOutput,
|
||||
toStructuredOutput: config.toStructuredOutput,
|
||||
_decode: Schema.decodeUnknownEffect(config.parameters),
|
||||
_encode: Schema.encodeEffect(config.success),
|
||||
_project: (parameters, callID, output) => project(config.toModelOutput, parameters, callID, output),
|
||||
_project: (parameters, callID, output) =>
|
||||
project(config.toModelOutput, config.toStructuredOutput, parameters, callID, output),
|
||||
_legacyResult: false,
|
||||
_definition: new ToolDefinition({
|
||||
name: "",
|
||||
|
|
@ -226,12 +237,13 @@ const toJsonSchema = (schema: Schema.Top): JsonSchema.JsonSchema => {
|
|||
|
||||
const project = (
|
||||
toModelOutput: ((input: ToolModelOutputInput<any, any>) => ReadonlyArray<ToolContent>) | undefined,
|
||||
toStructuredOutput: ((output: unknown) => unknown) | undefined,
|
||||
parameters: unknown,
|
||||
callID: ToolCallPart["id"],
|
||||
output: unknown,
|
||||
): ToolOutputType =>
|
||||
ToolOutput.make(
|
||||
output,
|
||||
toStructuredOutput?.(output) ?? output,
|
||||
toModelOutput?.({ callID, parameters, output }) ??
|
||||
(typeof output === "string" ? [toolText({ type: "text", text: output })] : []),
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue