From 3a15b915fc9ff464485e117f01a6a1ac6ec035ff Mon Sep 17 00:00:00 2001 From: shine1i Date: Mon, 2 Feb 2026 16:11:39 +0100 Subject: [PATCH] feat: add PDF attachment support using `unpdf`, extend attachment handling and runtime to process and extract text from PDFs --- studio/frontend/bun.lock | 3 ++ studio/frontend/package.json | 1 + studio/frontend/src/features/chat/adapter.ts | 20 +++++++---- .../src/features/chat/runtime-provider.tsx | 36 +++++++++++++++++++ 4 files changed, 54 insertions(+), 6 deletions(-) diff --git a/studio/frontend/bun.lock b/studio/frontend/bun.lock index 1417bea8db..9a888a7530 100644 --- a/studio/frontend/bun.lock +++ b/studio/frontend/bun.lock @@ -51,6 +51,7 @@ "tailwindcss": "^4.1.17", "tw-animate-css": "^1.4.0", "tw-shimmer": "^0.4.4", + "unpdf": "^1.4.0", "zustand": "^5.0.10", }, "devDependencies": { @@ -1809,6 +1810,8 @@ "universalify": ["universalify@2.0.1", "", {}, "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw=="], + "unpdf": ["unpdf@1.4.0", "", { "peerDependencies": { "@napi-rs/canvas": "^0.1.69" }, "optionalPeers": ["@napi-rs/canvas"] }, "sha512-TahIk0xdH/4jh/MxfclzU79g40OyxtP00VnEUZdEkJoYtXAHWLiir6t3FC6z3vDqQTzc2ZHcla6uEiVTNjejuA=="], + "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], "until-async": ["until-async@3.0.2", "", {}, "sha512-IiSk4HlzAMqTUseHHe3VhIGyuFmN90zMTpD3Z3y8jeQbzLIq500MVM7Jq2vUAnTKAFPJrqwkzr6PoTcPhGcOiw=="], diff --git a/studio/frontend/package.json b/studio/frontend/package.json index e09c04695b..8a13400ecc 100644 --- a/studio/frontend/package.json +++ b/studio/frontend/package.json @@ -59,6 +59,7 @@ "tailwindcss": "^4.1.17", "tw-animate-css": "^1.4.0", "tw-shimmer": "^0.4.4", + "unpdf": "^1.4.0", "zustand": "^5.0.10" }, "devDependencies": { diff --git a/studio/frontend/src/features/chat/adapter.ts b/studio/frontend/src/features/chat/adapter.ts index c4e103f62d..d1865dd47b 100644 --- a/studio/frontend/src/features/chat/adapter.ts +++ b/studio/frontend/src/features/chat/adapter.ts @@ -6,13 +6,21 @@ type RunMessages = Parameters[0]["messages"]; function makeBody(messages: RunMessages): string { return JSON.stringify({ - messages: messages.map((m) => ({ - role: m.role, - content: m.content + messages: messages.map((m) => { + const textParts = m.content .filter((c) => c.type === "text") - .map((c) => c.text) - .join(""), - })), + .map((c) => c.text); + + if ("attachments" in m && m.attachments?.length) { + for (const att of m.attachments) { + for (const part of att.content ?? []) { + if (part.type === "text") textParts.push(part.text); + } + } + } + + return { role: m.role, content: textParts.join("\n") }; + }), }); } diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index 8e4d242269..81604e0f87 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -1,8 +1,11 @@ import { + type AttachmentAdapter, AssistantRuntimeProvider, + type CompleteAttachment, CompositeAttachmentAdapter, ExportedMessageRepository, type ExportedMessageRepositoryItem, + type PendingAttachment, RuntimeAdapterProvider, SimpleImageAttachmentAdapter, SimpleTextAttachmentAdapter, @@ -19,10 +22,42 @@ import { } from "@assistant-ui/react"; import { createAssistantStream } from "assistant-stream"; import { type ReactElement, type ReactNode, useEffect, useMemo } from "react"; +import { extractText, getDocumentProxy } from "unpdf"; import { createStreamAdapter } from "./adapter"; import { db } from "./db"; import type { MessageRecord, ModelType } from "./types"; +class PDFAttachmentAdapter implements AttachmentAdapter { + accept = "application/pdf"; + + async add({ file }: { file: File }): Promise { + return { + id: crypto.randomUUID(), + type: "document", + name: file.name, + contentType: file.type, + file, + status: { type: "requires-action", reason: "composer-send" }, + }; + } + + async send(attachment: PendingAttachment): Promise { + const buffer = new Uint8Array(await attachment.file.arrayBuffer()); + const pdf = await getDocumentProxy(buffer); + const { text } = await extractText(pdf, { mergePages: true }); + return { + id: attachment.id, + type: "document", + name: attachment.name, + contentType: attachment.contentType, + content: [{ type: "text", text: `[PDF: ${attachment.name}]\n${text}` }], + status: { type: "complete" }, + }; + } + + async remove(): Promise {} +} + function toThreadMessage(m: MessageRecord): ThreadMessage { const base = { id: m.id, @@ -200,6 +235,7 @@ function ThreadHistoryProvider({ new CompositeAttachmentAdapter([ new SimpleImageAttachmentAdapter(), new SimpleTextAttachmentAdapter(), + new PDFAttachmentAdapter(), ]), [], );