diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index fa670ecda2..2b2356e847 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -45,6 +45,42 @@ function toOpenAIMessage(message: RunMessage): { }; } +function extractImageBase64(input: string): string | undefined { + if (!input) { + return undefined; + } + if (input.startsWith("data:")) { + const commaIndex = input.indexOf(","); + return commaIndex >= 0 ? input.slice(commaIndex + 1) : undefined; + } + return input; +} + +function findLatestUserImageBase64(messages: RunMessages): string | undefined { + for (let i = messages.length - 1; i >= 0; i -= 1) { + const message = messages[i]; + if (!message || message.role !== "user") { + continue; + } + + if ("attachments" in message && (message.attachments?.length ?? 0) > 0) { + for (const attachment of message.attachments ?? []) { + for (const part of attachment.content ?? []) { + if (part.type !== "image") { + continue; + } + const encoded = extractImageBase64(part.image); + if (encoded) { + return encoded; + } + } + } + } + } + + return undefined; +} + export function createOpenAIStreamAdapter(): ChatModelAdapter { return { async *run({ messages, abortSignal, unstable_threadId }) { @@ -67,6 +103,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { content: params.systemPrompt.trim(), }); } + const imageBase64 = findLatestUserImageBase64(messages); const threadKey = unstable_threadId || "__default"; let waitingFirstChunk = true; @@ -86,6 +123,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { max_tokens: params.maxTokens, top_k: params.topK, repetition_penalty: params.repetitionPenalty, + image_base64: imageBase64, }, abortSignal, ); diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index 7dcc6644e8..a94e25b67a 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -8,7 +8,6 @@ import { type PendingAttachment, RuntimeAdapterProvider, Suggestions, - SimpleImageAttachmentAdapter, SimpleTextAttachmentAdapter, type ThreadHistoryAdapter, type ThreadMessage, @@ -35,6 +34,55 @@ const DEFAULT_SUGGESTIONS = [ "Format a comparison of 3 databases as a markdown table with pros and cons", ]; +class VisionImageAdapter implements AttachmentAdapter { + accept = "image/jpeg,image/png,image/webp,image/gif"; + + async add({ file }: { file: File }): Promise { + const maxSize = 20 * 1024 * 1024; + if (file.size > maxSize) { + throw new Error("Image size exceeds 20MB limit"); + } + + return { + id: crypto.randomUUID(), + type: "image", + name: file.name, + contentType: file.type, + file, + status: { type: "requires-action", reason: "composer-send" }, + }; + } + + async send(attachment: PendingAttachment): Promise { + return { + id: attachment.id, + type: "image", + name: attachment.name, + contentType: attachment.contentType, + content: [ + { + type: "image", + image: await this.fileToBase64DataURL(attachment.file), + }, + ], + status: { type: "complete" }, + }; + } + + async remove(): Promise { + return Promise.resolve(); + } + + private async fileToBase64DataURL(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = () => reject(new Error("Failed to read image file")); + reader.readAsDataURL(file); + }); + } +} + class PDFAttachmentAdapter implements AttachmentAdapter { accept = "application/pdf"; @@ -276,7 +324,7 @@ function ThreadHistoryProvider({ const attachments = useMemo( () => new CompositeAttachmentAdapter([ - new SimpleImageAttachmentAdapter(), + new VisionImageAdapter(), new SimpleTextAttachmentAdapter(), new PDFAttachmentAdapter(), new DocxAttachmentAdapter(), diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 9d21c854af..f53bd6ca75 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -62,6 +62,7 @@ export interface OpenAIChatCompletionsRequest { max_tokens: number; top_k: number; repetition_penalty: number; + image_base64?: string; } export interface OpenAIChatDelta {