From 97f40bdc58c55bf8be66a58c53cf09dc7a2aa284 Mon Sep 17 00:00:00 2001 From: samit Date: Fri, 20 Feb 2026 22:14:27 -0800 Subject: [PATCH] Added dictate and add attachments feature --- .../src/features/chat/api/chat-adapter.ts | 9 + .../src/features/chat/shared-composer.tsx | 246 +++++++++++++++++- 2 files changed, 243 insertions(+), 12 deletions(-) diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 8e9e7df0e0..af8f10156f 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -65,6 +65,15 @@ function findLatestUserImageBase64(messages: RunMessages): string | undefined { continue; } + // Image in message.content (e.g. compare view appends content with image parts) + for (const part of message.content ?? []) { + if (part.type === "image" && "image" in part) { + const encoded = extractImageBase64(part.image); + if (encoded) return encoded; + } + } + + // Image in message.attachments (e.g. chat composer) if ("attachments" in message && (message.attachments?.length ?? 0) > 0) { for (const attachment of message.attachments ?? []) { for (const part of attachment.content ?? []) { diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index f849d3f5b7..5ddcb3947f 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -1,25 +1,103 @@ import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; import { Button } from "@/components/ui/button"; import { useAui } from "@assistant-ui/react"; -import { ArrowUpIcon, SquareIcon } from "lucide-react"; +import { ArrowUpIcon, MicIcon, PlusIcon, SquareIcon, XIcon } from "lucide-react"; import { type KeyboardEvent, type MutableRefObject, type ReactElement, type ReactNode, createContext, + useCallback, useContext, useEffect, useRef, useState, } from "react"; +export type CompareMessagePart = + | { type: "text"; text: string } + | { type: "image"; image: string }; + export interface CompareHandle { - append: (content: { type: "text"; text: string }[]) => void; + append: (content: CompareMessagePart[]) => void; cancel: () => void; isRunning: () => boolean; } +const IMAGE_ACCEPT = "image/jpeg,image/png,image/webp,image/gif"; +const MAX_IMAGE_SIZE = 20 * 1024 * 1024; + +function fileToBase64DataURL(file: File): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = () => reject(new Error("Failed to read image file")); + reader.readAsDataURL(file); + }); +} + +function useDictation( + textareaRef: React.RefObject, + setText: (value: string | ((prev: string) => string)) => void, +) { + const [isDictating, setIsDictating] = useState(false); + const recognitionRef = useRef(null); + + const start = useCallback(() => { + const SpeechRecognitionAPI = + typeof window !== "undefined" && + (window.SpeechRecognition ?? (window as unknown as { webkitSpeechRecognition?: typeof SpeechRecognition }).webkitSpeechRecognition); + if (!SpeechRecognitionAPI) { + return; + } + const recognition = new SpeechRecognitionAPI() as SpeechRecognition; + recognition.continuous = true; + recognition.interimResults = true; + recognition.lang = "en-US"; + recognition.onresult = (event: SpeechRecognitionEvent) => { + const last = event.resultIndex; + const result = event.results[last]; + if (!result?.isFinal) return; + const transcript = result[0]?.transcript?.trim(); + if (transcript) { + setText((prev) => (prev ? `${prev} ${transcript}` : transcript)); + } + }; + recognition.onerror = () => { + setIsDictating(false); + }; + recognition.onend = () => { + setIsDictating(false); + }; + recognition.start(); + recognitionRef.current = recognition; + setIsDictating(true); + }, [setText]); + + const stop = useCallback(() => { + if (recognitionRef.current) { + recognitionRef.current.stop(); + recognitionRef.current = null; + } + setIsDictating(false); + }, []); + + useEffect(() => { + return () => { + if (recognitionRef.current) { + recognitionRef.current.abort(); + } + }; + }, []); + + const supported = + typeof window !== "undefined" && + !!(window.SpeechRecognition ?? (window as unknown as { webkitSpeechRecognition?: unknown }).webkitSpeechRecognition); + + return { isDictating, start, stop, supported }; +} + export type CompareHandles = MutableRefObject>; const CompareHandlesContext = createContext(null); @@ -66,6 +144,37 @@ export function RegisterCompareHandle({ return null; } +type PendingImage = { id: string; file: File }; + +function PendingImageThumb({ + file, + onRemove, +}: { + file: File; + onRemove: () => void; +}): ReactElement { + const [src, setSrc] = useState(null); + useEffect(() => { + const url = URL.createObjectURL(file); + setSrc(url); + return () => URL.revokeObjectURL(url); + }, [file]); + if (!src) return
; + return ( +
+ {file.name} + +
+ ); +} + export function SharedComposer({ handlesRef, }: { @@ -73,7 +182,15 @@ export function SharedComposer({ }): ReactElement { const [text, setText] = useState(""); const [running, setRunning] = useState(false); + const [pendingImages, setPendingImages] = useState([]); + const [dragging, setDragging] = useState(false); const textareaRef = useRef(null); + const fileInputRef = useRef(null); + + const { isDictating, start: startDictation, stop: stopDictation, supported: dictationSupported } = useDictation( + textareaRef, + setText, + ); useEffect(() => { const id = setInterval(() => { @@ -84,23 +201,50 @@ export function SharedComposer({ return () => clearInterval(id); }, [handlesRef]); - function send() { - const msg = text.trim(); - if (!msg) { - return; + const addFiles = useCallback((files: FileList | null) => { + if (!files?.length) return; + const next: PendingImage[] = []; + for (let i = 0; i < files.length; i++) { + const file = files[i]; + if (!file?.type.match(/^image\/(jpeg|png|webp|gif)$/i)) continue; + if (file.size > MAX_IMAGE_SIZE) continue; + next.push({ id: crypto.randomUUID(), file }); } + setPendingImages((prev) => [...prev, ...next]); + }, []); + + const removePendingImage = useCallback((id: string) => { + setPendingImages((prev) => prev.filter((p) => p.id !== id)); + }, []); + + async function send() { + const msg = text.trim(); + if (!msg && pendingImages.length === 0) return; + + const content: CompareMessagePart[] = []; + for (const { file } of pendingImages) { + try { + const image = await fileToBase64DataURL(file); + content.push({ type: "image", image }); + } catch { + // skip failed image + } + } + if (msg) { + content.push({ type: "text", text: msg }); + } + if (content.length === 0) return; - const content: { type: "text"; text: string }[] = [ - { type: "text", text: msg }, - ]; for (const handle of Object.values(handlesRef.current)) { handle.append(content); } setText(""); + setPendingImages([]); textareaRef.current?.focus(); } function stop() { + if (isDictating) stopDictation(); for (const handle of Object.values(handlesRef.current)) { handle.cancel(); } @@ -115,8 +259,33 @@ export function SharedComposer({ } } + const canSend = (text.trim().length > 0 || pendingImages.length > 0) && !running; + return ( -
+
{ + e.preventDefault(); + setDragging(true); + }} + onDragLeave={() => setDragging(false)} + onDrop={(e) => { + e.preventDefault(); + setDragging(false); + addFiles(e.dataTransfer.files); + }} + > + {pendingImages.length > 0 && ( +
+ {pendingImages.map(({ id, file }) => ( + removePendingImage(id)} + /> + ))} +
+ )}