Merge pull request #78 from unslothai/feature/vision-capabilities-chat

feat(chat): add vision image attachments for OpenAI-compatible chat

studio/frontend/src/features/chat/api/chat-adapter.ts

@@ -45,6 +45,42 @@ function toOpenAIMessage(message: RunMessage): {
   };
 }
 
+function extractImageBase64(input: string): string | undefined {
+  if (!input) {
+    return undefined;
+  }
+  if (input.startsWith("data:")) {
+    const commaIndex = input.indexOf(",");
+    return commaIndex >= 0 ? input.slice(commaIndex + 1) : undefined;
+  }
+  return input;
+}
+
+function findLatestUserImageBase64(messages: RunMessages): string | undefined {
+  for (let i = messages.length - 1; i >= 0; i -= 1) {
+    const message = messages[i];
+    if (!message || message.role !== "user") {
+      continue;
+    }
+
+    if ("attachments" in message && (message.attachments?.length ?? 0) > 0) {
+      for (const attachment of message.attachments ?? []) {
+        for (const part of attachment.content ?? []) {
+          if (part.type !== "image") {
+            continue;
+          }
+          const encoded = extractImageBase64(part.image);
+          if (encoded) {
+            return encoded;
+          }
+        }
+      }
+    }
+  }
+
+  return undefined;
+}
+
 export function createOpenAIStreamAdapter(): ChatModelAdapter {
   return {
     async *run({ messages, abortSignal, unstable_threadId }) {
@@ -67,6 +103,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
           content: params.systemPrompt.trim(),
         });
       }
+      const imageBase64 = findLatestUserImageBase64(messages);
 
       const threadKey = unstable_threadId || "__default";
       let waitingFirstChunk = true;
@@ -86,6 +123,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
             max_tokens: params.maxTokens,
             top_k: params.topK,
             repetition_penalty: params.repetitionPenalty,
+            image_base64: imageBase64,
           },
           abortSignal,
         );

studio/frontend/src/features/chat/runtime-provider.tsx

@@ -8,7 +8,6 @@ import {
   type PendingAttachment,
   RuntimeAdapterProvider,
   Suggestions,
-  SimpleImageAttachmentAdapter,
   SimpleTextAttachmentAdapter,
   type ThreadHistoryAdapter,
   type ThreadMessage,
@@ -35,6 +34,55 @@ const DEFAULT_SUGGESTIONS = [
   "Format a comparison of 3 databases as a markdown table with pros and cons",
 ];
 
+class VisionImageAdapter implements AttachmentAdapter {
+  accept = "image/jpeg,image/png,image/webp,image/gif";
+
+  async add({ file }: { file: File }): Promise<PendingAttachment> {
+    const maxSize = 20 * 1024 * 1024;
+    if (file.size > maxSize) {
+      throw new Error("Image size exceeds 20MB limit");
+    }
+
+    return {
+      id: crypto.randomUUID(),
+      type: "image",
+      name: file.name,
+      contentType: file.type,
+      file,
+      status: { type: "requires-action", reason: "composer-send" },
+    };
+  }
+
+  async send(attachment: PendingAttachment): Promise<CompleteAttachment> {
+    return {
+      id: attachment.id,
+      type: "image",
+      name: attachment.name,
+      contentType: attachment.contentType,
+      content: [
+        {
+          type: "image",
+          image: await this.fileToBase64DataURL(attachment.file),
+        },
+      ],
+      status: { type: "complete" },
+    };
+  }
+
+  async remove(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  private async fileToBase64DataURL(file: File): Promise<string> {
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader();
+      reader.onload = () => resolve(reader.result as string);
+      reader.onerror = () => reject(new Error("Failed to read image file"));
+      reader.readAsDataURL(file);
+    });
+  }
+}
+
 class PDFAttachmentAdapter implements AttachmentAdapter {
   accept = "application/pdf";
 
@@ -276,7 +324,7 @@ function ThreadHistoryProvider({
   const attachments = useMemo(
     () =>
       new CompositeAttachmentAdapter([
-        new SimpleImageAttachmentAdapter(),
+        new VisionImageAdapter(),
         new SimpleTextAttachmentAdapter(),
         new PDFAttachmentAdapter(),
         new DocxAttachmentAdapter(),

studio/frontend/src/features/chat/types/api.ts

@@ -62,6 +62,7 @@ export interface OpenAIChatCompletionsRequest {
   max_tokens: number;
   top_k: number;
   repetition_penalty: number;
+  image_base64?: string;
 }
 
 export interface OpenAIChatDelta {