opencode/packages/ai/test/provider/pdf.recorded.test.ts

import { describe, expect } from "bun:test"
import { Effect, Schema, Stream } from "effect"
import { LLM, LLMResponse, Message, ToolDefinition, type Model } from "../../src"
import { AmazonBedrock, Anthropic, Google, OpenAI, XAI } from "../../src/providers"
import { LLMClient } from "../../src/route"
import { Tool } from "../../src/tool"
import { runTools } from "../lib/tool-runtime"
import { recordedTests } from "../recorded-test"

const CODE = "ORCHID-7391"
const PDF =
  "JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvTWVkaWFCb3ggWzAgMCA2MTIgNzkyXSAvUmVzb3VyY2VzIDw8IC9Gb250IDw8IC9GMSA1IDAgUiA+PiA+PiAvQ29udGVudHMgNCAwIFIgPj4KZW5kb2JqCjQgMCBvYmoKPDwgL0xlbmd0aCA3NSA+PgpzdHJlYW0KQlQKL0YxIDE4IFRmCjcyIDcyMCBUZAooUERGIGNhc3NldHRlIHZlcmlmaWNhdGlvbiBjb2RlOiBPUkNISUQtNzM5MSkgVGoKRVQKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCjw8IC9UeXBlIC9Gb250IC9TdWJ0eXBlIC9UeXBlMSAvQmFzZUZvbnQgL0hlbHZldGljYSA+PgplbmRvYmoKeHJlZgowIDYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMDA5IDAwMDAwIG4gCjAwMDAwMDAwNTggMDAwMDAgbiAKMDAwMDAwMDExNSAwMDAwMCBuIAowMDAwMDAwMjQxIDAwMDAwIG4gCjAwMDAwMDAzNjUgMDAwMDAgbiAKdHJhaWxlcgo8PCAvU2l6ZSA2IC9Sb290IDEgMCBSID4+CnN0YXJ0eHJlZgo0MzUKJSVFT0YK"

const openai = OpenAI.configure({ apiKey: process.env.OPENAI_API_KEY ?? "fixture" })
const anthropic = Anthropic.configure({ apiKey: process.env.ANTHROPIC_API_KEY ?? "fixture" })
const google = Google.configure({ apiKey: process.env.GOOGLE_API_KEY ?? "fixture" })
const xai = XAI.configure({ apiKey: process.env.XAI_API_KEY ?? "fixture" })
const bedrock = AmazonBedrock.configure({
  apiKey: process.env.AWS_BEDROCK_API_KEY ?? "fixture",
  region: process.env.AWS_REGION ?? "us-east-1",
})

const targets: ReadonlyArray<{
  readonly id: string
  readonly name: string
  readonly provider: string
  readonly protocol: string
  readonly requires: string
  readonly filename: string
  readonly maxTokens: number
  readonly model: Model
}> = [
  {
    id: "openai",
    name: "OpenAI Responses gpt-4o-mini",
    provider: "openai",
    protocol: "openai-responses",
    requires: "OPENAI_API_KEY",
    filename: "verification.pdf",
    maxTokens: 40,
    model: openai.responses("gpt-4o-mini"),
  },
  {
    id: "anthropic",
    name: "Anthropic Haiku 4.5",
    provider: "anthropic",
    protocol: "anthropic-messages",
    requires: "ANTHROPIC_API_KEY",
    filename: "verification.pdf",
    maxTokens: 40,
    model: anthropic.model("claude-haiku-4-5-20251001"),
  },
  {
    id: "gemini",
    name: "Gemini 3.5 Flash",
    provider: "google",
    protocol: "gemini",
    requires: "GOOGLE_API_KEY",
    filename: "verification.pdf",
    maxTokens: 256,
    model: google.model("gemini-3.5-flash"),
  },
  {
    id: "xai",
    name: "xAI Grok 4.5",
    provider: "xai",
    protocol: "openai-responses",
    requires: "XAI_API_KEY",
    filename: "verification.pdf",
    maxTokens: 40,
    model: xai.responses("grok-4.5"),
  },
  {
    id: "bedrock",
    name: "Bedrock Claude Haiku 4.5",
    provider: "amazon-bedrock",
    protocol: "bedrock-converse",
    requires: "AWS_BEDROCK_API_KEY",
    filename: "verification",
    maxTokens: 40,
    model: bedrock.model("us.anthropic.claude-haiku-4-5-20251001-v1:0"),
  },
]

const recorded = recordedTests({ prefix: "pdf", tags: ["pdf"] })
const prompt = "Return only the verification code from the PDF."
const readPdf = ToolDefinition.make({
  name: "read_pdf",
  description: "Read the attached PDF.",
  inputSchema: { type: "object", properties: {}, additionalProperties: false },
})
const readPdfRuntime = Tool.make({
  description: readPdf.description,
  parameters: Schema.Struct({ path: Schema.String }),
  success: Schema.String,
  execute: () => Effect.succeed("PDF read successfully"),
  toModelOutput: () => [
    { type: "text", text: "PDF read successfully" },
    {
      type: "file",
      uri: `data:application/pdf;base64,${PDF}`,
      mime: "application/pdf",
      name: "verification.pdf",
    },
  ],
})

const expectCode = (response: LLMResponse) => {
  expect(response.finishReason.normalized).toBe("stop")
  expect(response.text.toUpperCase()).toContain(CODE)
}

describe("PDF recorded", () => {
  for (const target of targets) {
    recorded.effect.with(
      `reads a user PDF with ${target.name}`,
      {
        id: `${target.id}-user-input`,
        provider: target.provider,
        protocol: target.protocol,
        requires: [target.requires],
        tags: ["user-input"],
      },
      Effect.gen(function* () {
        expectCode(
          yield* LLMClient.generate(
            LLM.request({
              id: `recorded_pdf_${target.id}_user_input`,
              model: target.model,
              cache: "none",
              generation: { maxTokens: target.maxTokens, temperature: 0 },
              messages: [
                Message.user([
                  { type: "media", mediaType: "application/pdf", data: PDF, filename: target.filename },
                  { type: "text", text: prompt },
                ]),
              ],
            }),
          ),
        )
      }),
    )

    recorded.effect.with(
      `reads a PDF tool result with ${target.name}`,
      {
        id: `${target.id}-tool-result`,
        provider: target.provider,
        protocol: target.protocol,
        requires: [target.requires],
        tags: ["tool", "tool-result"],
      },
      Effect.gen(function* () {
        if (target.id === "gemini") {
          const events = Array.from(
            yield* runTools({
              request: LLM.request({
                id: "recorded_pdf_gemini_tool_result",
                model: target.model,
                system:
                  "Call read_pdf exactly once with path verification.pdf, then reply only with the verification code from its PDF.",
                prompt: "Use read_pdf with path verification.pdf and return the verification code.",
                cache: "none",
                generation: { maxTokens: target.maxTokens, temperature: 0 },
              }),
              tools: { read_pdf: readPdfRuntime },
            }).pipe(Stream.runCollect),
          )
          expect(events.at(-1)).toMatchObject({ type: "finish", reason: { normalized: "stop" } })
          expect(LLMResponse.text({ events }).toUpperCase()).toContain(CODE)
          return
        }

        expectCode(
          yield* LLMClient.generate(
            LLM.request({
              id: `recorded_pdf_${target.id}_tool_result`,
              model: target.model,
              system: "Read the PDF returned by the tool and follow the user's response format exactly.",
              cache: "none",
              generation: { maxTokens: target.maxTokens, temperature: 0 },
              messages: [
                Message.user(prompt),
                Message.assistant([{ type: "tool-call", id: "call_pdf_1", name: readPdf.name, input: {} }]),
                Message.tool({
                  id: "call_pdf_1",
                  name: readPdf.name,
                  resultType: "content",
                  result: [
                    { type: "text", text: "PDF read successfully" },
                    {
                      type: "file",
                      uri: `data:application/pdf;base64,${PDF}`,
                      mime: "application/pdf",
                      name: target.filename,
                    },
                  ],
                }),
              ],
              tools: [readPdf],
            }),
          ),
        )
      }),
    )
  }
})