ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ai): support PDF inputs (#38253)

Browse source
This commit is contained in:
Aiden Cline 2026-07-22 13:42:29 -05:00 • committed by GitHub
commit 2271f9b222
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 1063 additions and 96 deletions
Download patch file Download diff file Expand all files Collapse all files

65
packages/ai/src/protocols/anthropic-messages.ts
View file

@ -27,6 +27,7 @@ import { ToolSchemaProjection } from "./utils/tool-schema"
import { ToolStream } from "./utils/tool-stream"
const ADAPTER = "anthropic-messages"
const MEDIA_MIMES = new Set<string>([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES])
export const DEFAULT_BASE_URL = "https://api.anthropic.com/v1"
export const PATH = "/messages"
@ -56,6 +57,17 @@ const AnthropicImageBlock = Schema.Struct({
})
type AnthropicImageBlock = Schema.Schema.Type<typeof AnthropicImageBlock>
const AnthropicDocumentBlock = Schema.Struct({
type: Schema.tag("document"),
source: Schema.Struct({
type: Schema.tag("base64"),
media_type: Schema.Literal("application/pdf"),
data: Schema.String,
}),
cache_control: Schema.optional(AnthropicCacheControl),
})
type AnthropicDocumentBlock = Schema.Schema.Type<typeof AnthropicDocumentBlock>
const AnthropicThinkingBlock = Schema.Struct({
type: Schema.tag("thinking"),
thinking: Schema.String,
@ -101,13 +113,10 @@ const AnthropicServerToolResultBlock = Schema.Struct({
})
type AnthropicServerToolResultBlock = Schema.Schema.Type<typeof AnthropicServerToolResultBlock>
// Anthropic accepts either a plain string or an ordered array of text/image
// blocks inside `tool_result.content`. The array form is required when a tool
// returns image bytes (screenshot, image search, etc.) so they can be passed
// to the model as proper image inputs instead of being JSON-stringified into
// the prompt — which silently inflates context by megabytes and can push the
// conversation over the model's token limit.
const AnthropicToolResultContent = Schema.Union([AnthropicTextBlock, AnthropicImageBlock])
// Anthropic accepts either a plain string or an ordered array of text, image, and
// document blocks inside `tool_result.content`. The array form keeps media as native
// model input instead of JSON-stringifying base64 into prompt text.
const AnthropicToolResultContent = Schema.Union([AnthropicTextBlock, AnthropicImageBlock, AnthropicDocumentBlock])
const AnthropicToolResultBlock = Schema.Struct({
type: Schema.tag("tool_result"),
@ -117,7 +126,12 @@ const AnthropicToolResultBlock = Schema.Struct({
cache_control: Schema.optional(AnthropicCacheControl),
})
const AnthropicUserBlock = Schema.Union([AnthropicTextBlock, AnthropicImageBlock, AnthropicToolResultBlock])
const AnthropicUserBlock = Schema.Union([
AnthropicTextBlock,
AnthropicImageBlock,
AnthropicDocumentBlock,
AnthropicToolResultBlock,
])
type AnthropicUserBlock = Schema.Schema.Type<typeof AnthropicUserBlock>
const AnthropicAssistantBlock = Schema.Union([
AnthropicTextBlock,
@ -319,12 +333,17 @@ const lowerServerToolResult = Effect.fn("AnthropicMessages.lowerServerToolResult
return { type: wireType, tool_use_id: part.id, content: part.result.value } satisfies AnthropicServerToolResultBlock
})
const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: MediaPart) {
const media = yield* ProviderShared.validateMedia(
"Anthropic Messages",
part,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
const lowerMedia = Effect.fn("AnthropicMessages.lowerMedia")(function* (part: MediaPart) {
const media = yield* ProviderShared.validateMedia("Anthropic Messages", part, MEDIA_MIMES)
if (media.mime === "application/pdf")
return {
type: "document" as const,
source: {
type: "base64" as const,
media_type: "application/pdf" as const,
data: media.base64,
},
} satisfies AnthropicDocumentBlock
return {
type: "image" as const,
source: {
@ -335,25 +354,13 @@ const lowerImage = Effect.fn("AnthropicMessages.lowerImage")(function* (part: Me
} satisfies AnthropicImageBlock
})
// Tool results may carry structured text/images. Keep media as provider-native
// Tool results may carry structured text, images, and documents. Keep media as provider-native
// content instead of JSON-stringifying base64 into a prompt string.
const lowerToolResultContentItem = Effect.fn("AnthropicMessages.lowerToolResultContentItem")(function* (
item: ToolContent,
) {
if (item.type === "text") return { type: "text" as const, text: item.text } satisfies AnthropicTextBlock
const media = yield* ProviderShared.validateToolFile(
"Anthropic Messages",
item,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return {
type: "image" as const,
source: {
type: "base64" as const,
media_type: media.mime,
data: media.base64,
},
} satisfies AnthropicImageBlock
return yield* lowerMedia({ type: "media", mediaType: item.mime, data: item.uri, filename: item.name })
})
const lowerToolResultContent = Effect.fn("AnthropicMessages.lowerToolResultContent")(function* (part: ToolResultPart) {
@ -445,7 +452,7 @@ const lowerMessages = Effect.fn("AnthropicMessages.lowerMessages")(function* (
continue
}
if (part.type === "media") {
content.push(yield* lowerImage(part))
content.push(yield* lowerMedia(part))
continue
}
return yield* ProviderShared.unsupportedContent("Anthropic Messages", "user", ["text", "media"])

3
packages/ai/src/protocols/bedrock-converse.ts
View file

@ -52,6 +52,7 @@ const BedrockToolResultContentItem = Schema.Union([
Schema.Struct({ text: Schema.String }),
Schema.Struct({ json: Schema.Unknown }),
BedrockMedia.ImageBlock,
BedrockMedia.DocumentBlock,
])
const BedrockToolResultBlock = Schema.Struct({
@ -283,8 +284,6 @@ const lowerToolResultContent = Effect.fn("BedrockConverse.lowerToolResultContent
data: item.uri,
filename: item.name,
})
if (!("image" in media))
return yield* ProviderShared.invalidRequest("Bedrock Converse only supports image media in tool results")
content.push(media)
}
return content

33
packages/ai/src/protocols/gemini.ts
View file

@ -41,9 +41,11 @@ const GeminiInlineDataPart = Schema.Struct({
data: Schema.String,
}),
})
type GeminiInlineDataPart = Schema.Schema.Type<typeof GeminiInlineDataPart>
const GeminiFunctionCallPart = Schema.Struct({
functionCall: Schema.Struct({
id: Schema.optional(Schema.String),
name: Schema.String,
args: Schema.Unknown,
}),
@ -52,8 +54,10 @@ const GeminiFunctionCallPart = Schema.Struct({
const GeminiFunctionResponsePart = Schema.Struct({
functionResponse: Schema.Struct({
id: Schema.optional(Schema.String),
name: Schema.String,
response: Schema.Unknown,
parts: Schema.optional(Schema.Array(GeminiInlineDataPart)),
}),
})
@ -197,8 +201,13 @@ const thoughtSignature = (providerMetadata: ProviderMetadata | undefined) => {
: undefined
}
const functionCallId = (providerMetadata: ProviderMetadata | undefined) => {
const google = providerMetadata?.google
return ProviderShared.isRecord(google) && typeof google.functionCallId === "string" ? google.functionCallId : undefined
}
const lowerToolCall = (part: ToolCallPart) => ({
functionCall: { name: part.name, args: part.input },
functionCall: { id: functionCallId(part.providerMetadata), name: part.name, args: part.input },
thoughtSignature: thoughtSignature(part.providerMetadata),
})
@ -255,6 +264,7 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
if (part.result.type !== "content") {
parts.push({
functionResponse: {
id: functionCallId(part.providerMetadata),
name: part.name,
response: {
name: part.name,
@ -266,20 +276,23 @@ const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMR
}
const content: ReadonlyArray<ToolContent> = part.result.value
const text = content.filter((item) => item.type === "text").map((item) => item.text)
const media: GeminiInlineDataPart[] = []
for (const item of content) {
if (item.type === "text") continue
const value = yield* ProviderShared.validateToolFile("Gemini", item, MEDIA_MIMES)
media.push({ inlineData: { mimeType: value.mime, data: value.base64 } })
}
parts.push({
functionResponse: {
id: functionCallId(part.providerMetadata),
name: part.name,
response: {
name: part.name,
content: text.join("\n"),
},
parts: media.length > 0 ? media : undefined,
},
})
for (const item of content) {
if (item.type === "text") continue
const media = yield* ProviderShared.validateToolFile("Gemini", item, MEDIA_MIMES)
parts.push({ inlineData: { mimeType: media.mime, data: media.base64 } })
}
}
contents.push({ role: "user", parts })
}
@ -441,6 +454,10 @@ const step = (state: ParserState, event: GeminiEvent) => {
if ("functionCall" in part) {
const input = part.functionCall.args
const id = `tool_${nextToolCallId++}`
const metadata = {
...(part.functionCall.id === undefined ? {} : { functionCallId: part.functionCall.id }),
...(part.thoughtSignature === undefined ? {} : { thoughtSignature: part.thoughtSignature }),
}
lifecycle = Lifecycle.reasoningEnd(
lifecycle,
events,
@ -453,9 +470,7 @@ const step = (state: ParserState, event: GeminiEvent) => {
id,
name: part.functionCall.name,
input,
providerMetadata: part.thoughtSignature
? googleMetadata({ thoughtSignature: part.thoughtSignature })
: undefined,
providerMetadata: Object.keys(metadata).length > 0 ? googleMetadata(metadata) : undefined,
}),
)
hasToolCalls = true

103
packages/ai/src/protocols/openai-responses.ts
View file

@ -11,6 +11,7 @@ import {
type FinishReason,
type JsonSchema,
type LLMRequest,
type MediaPart,
type ProviderMetadata,
type ReasoningPart,
type TextPart,
@ -28,6 +29,7 @@ import { ToolStream } from "./utils/tool-stream"
import { OpenAIImage } from "./utils/openai-image"
const ADAPTER = "openai-responses"
const MEDIA_MIMES = new Set<string>([...ProviderShared.IMAGE_MIMES, ...ProviderShared.PDF_MIMES])
export const DEFAULT_BASE_URL = "https://api.openai.com/v1"
export const PATH = "/responses"
@ -42,7 +44,17 @@ const OpenAIResponsesInputImage = Schema.Struct({
type: Schema.tag("input_image"),
image_url: Schema.String,
})
const OpenAIResponsesInputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage])
const OpenAIResponsesInputFile = Schema.Struct({
type: Schema.tag("input_file"),
filename: Schema.String,
file_data: Schema.String,
mime_type: Schema.optional(Schema.String),
})
const OpenAIResponsesInputContent = Schema.Union([
OpenAIResponsesInputText,
OpenAIResponsesInputImage,
OpenAIResponsesInputFile,
])
type OpenAIResponsesInputContent = Schema.Schema.Type<typeof OpenAIResponsesInputContent>
const OpenAIResponsesOutputText = Schema.Struct({
@ -68,9 +80,13 @@ const OpenAIResponsesItemReference = Schema.Struct({
})
// `function_call_output.output` accepts either a plain string or an ordered
// array of content items so tools can return images in addition to text.
// array of content items so tools can return images and files in addition to text.
// https://platform.openai.com/docs/api-reference/responses/object
const OpenAIResponsesFunctionCallOutputContent = Schema.Union([OpenAIResponsesInputText, OpenAIResponsesInputImage])
const OpenAIResponsesFunctionCallOutputContent = Schema.Union([
OpenAIResponsesInputText,
OpenAIResponsesInputImage,
OpenAIResponsesInputFile,
])
const OpenAIResponsesFunctionCallOutput = Schema.Union([
Schema.String,
@ -343,42 +359,58 @@ const hostedToolItemID = (part: ToolResultPart) => {
: undefined
}
const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function* (
part: LLMRequest["messages"][number]["content"][number],
) {
if (part.type === "text") return { type: "input_text" as const, text: part.text }
if (part.type === "media") {
const media = yield* ProviderShared.validateMedia(
"OpenAI Responses",
part,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return { type: "input_image" as const, image_url: media.dataUrl }
const lowerMedia = Effect.fn("OpenAIResponses.lowerMedia")(function* (part: MediaPart, provider: string) {
const media = yield* ProviderShared.validateMedia("OpenAI Responses", part, MEDIA_MIMES)
if (media.mime === "application/pdf") {
// xAI models inline bytes and MIME separately; OpenAI uses a data URL in file_data.
if (provider === "xai")
return {
type: "input_file" as const,
filename: part.filename ?? "document.pdf",
file_data: media.base64,
mime_type: media.mime,
}
return {
type: "input_file" as const,
filename: part.filename ?? "document.pdf",
file_data: media.dataUrl,
}
}
return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"])
})
// Tool results may carry structured text/images. Keep media as provider-native
// content instead of JSON-stringifying base64 into a prompt string.
const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultContentItem")(function* (
item: ToolContent,
) {
if (item.type === "text") return { type: "input_text" as const, text: item.text }
const media = yield* ProviderShared.validateToolFile(
"OpenAI Responses",
item,
new Set<string>(ProviderShared.IMAGE_MIMES),
)
return { type: "input_image" as const, image_url: media.dataUrl }
})
const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")(function* (part: ToolResultPart) {
const lowerUserContent = Effect.fn("OpenAIResponses.lowerUserContent")(function* (
part: LLMRequest["messages"][number]["content"][number],
provider: string,
) {
if (part.type === "text") return { type: "input_text" as const, text: part.text }
if (part.type === "media") return yield* lowerMedia(part, provider)
return yield* ProviderShared.unsupportedContent("OpenAI Responses", "user", ["text", "media"])
})
// Tool results may carry structured text, images, and files. Keep media as provider-native
// content instead of JSON-stringifying base64 into a prompt string.
const lowerToolResultContentItem = Effect.fn("OpenAIResponses.lowerToolResultContentItem")(function* (
item: ToolContent,
provider: string,
) {
if (item.type === "text") return { type: "input_text" as const, text: item.text }
return yield* lowerMedia(
{ type: "media", mediaType: item.mime, data: item.uri, filename: item.name },
provider,
)
})
const lowerToolResultOutput = Effect.fn("OpenAIResponses.lowerToolResultOutput")(function* (
part: ToolResultPart,
provider: string,
) {
// Text/json/error results are encoded as a plain string for backward
// compatibility with existing cassettes and provider expectations.
if (part.result.type !== "content") return ProviderShared.toolResultText(part)
// Preserve the narrowed array element type when compiled through a consumer package.
const content: ReadonlyArray<ToolContent> = part.result.value
return yield* Effect.forEach(content, lowerToolResultContentItem)
return yield* Effect.forEach(content, (item) => lowerToolResultContentItem(item, provider))
})
const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (request: LLMRequest) {
@ -401,7 +433,10 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ
}
if (message.role === "user") {
input.push({ role: "user", content: yield* Effect.forEach(message.content, lowerUserContent) })
input.push({
role: "user",
content: yield* Effect.forEach(message.content, (part) => lowerUserContent(part, request.model.provider)),
})
continue
}
@ -460,7 +495,9 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ
const content: ReadonlyArray<ToolContent> = part.result.value
input.push({
role: "user",
content: yield* Effect.forEach(content, lowerToolResultContentItem),
content: yield* Effect.forEach(content, (item) =>
lowerToolResultContentItem(item, request.model.provider),
),
})
}
if (itemID) hostedToolReferences.add(itemID)
@ -483,7 +520,7 @@ const lowerMessages = Effect.fn("OpenAIResponses.lowerMessages")(function* (requ
input.push({
type: "function_call_output",
call_id: part.id,
output: yield* lowerToolResultOutput(part),
output: yield* lowerToolResultOutput(part, request.model.provider),
})
}
}

3
packages/ai/src/protocols/shared.ts
View file

@ -158,7 +158,8 @@ export const parseToolInput = (route: string, name: string, raw: string) =>
export const IMAGE_MIMES = ["image/png", "image/jpeg", "image/gif", "image/webp"] as const
export const VIDEO_MIMES = ["video/mp4", "video/webm", "video/quicktime"] as const
export const AUDIO_MIMES = ["audio/wav", "audio/mp3", "audio/aiff", "audio/aac", "audio/ogg", "audio/flac"] as const
export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES] as const
export const PDF_MIMES = ["application/pdf"] as const
export const MEDIA_MIMES = [...IMAGE_MIMES, ...VIDEO_MIMES, ...AUDIO_MIMES, ...PDF_MIMES] as const
export const MAX_MEDIA_ENCODED_BYTES = 28 * 1024 * 1024
export const MAX_MEDIA_DECODED_BYTES = 20 * 1024 * 1024

8
packages/ai/src/protocols/utils/bedrock-media.ts
View file

@ -49,10 +49,10 @@ const DOCUMENT_FORMATS = {
"text/markdown": "md",
} as const satisfies Record<string, DocumentFormat>
const documentBlock = (part: MediaPart, format: DocumentFormat, bytes: string): DocumentBlock => ({
const documentBlock = (name: string, format: DocumentFormat, bytes: string): DocumentBlock => ({
document: {
format,
name: part.filename ?? `document.${format}`,
name,
source: { bytes },
},
})
@ -77,12 +77,14 @@ export const lower = Effect.fn("BedrockMedia.lower")(function* (part: MediaPart)
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support image media type ${part.mediaType}`)
const documentFormat = DOCUMENT_FORMATS[mime as keyof typeof DOCUMENT_FORMATS]
if (documentFormat) {
if (!part.filename)
return yield* ProviderShared.invalidRequest("Bedrock Converse document media requires a filename")
const media = yield* ProviderShared.validateMedia(
"Bedrock Converse",
part,
new Set<string>(Object.keys(DOCUMENT_FORMATS)),
)
return documentBlock(part, documentFormat, media.base64)
return documentBlock(part.filename, documentFormat, media.base64)
}
return yield* ProviderShared.invalidRequest(`Bedrock Converse does not support media type ${part.mediaType}`)
})