diff --git a/studio/frontend/src/components/assistant-ui/thread.tsx b/studio/frontend/src/components/assistant-ui/thread.tsx index 9b502a5000..93cf162ca7 100644 --- a/studio/frontend/src/components/assistant-ui/thread.tsx +++ b/studio/frontend/src/components/assistant-ui/thread.tsx @@ -97,9 +97,11 @@ import { ThreadDocumentsBar } from "@/features/rag/components/thread-documents-b import { KnowledgeBaseComposerButton } from "@/features/rag/components/knowledge-base-composer-button"; import { DocumentPreviewMount } from "@/features/rag/components/document-preview-mount"; import { useUserProfileStore } from "@/features/profile/stores/user-profile-store"; +import { useVoiceSettingsStore } from "@/features/settings/stores/voice-settings-store"; import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; import { isTauri } from "@/lib/api-base"; import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { MicIcon } from "@/lib/mic-icon"; import { toast } from "@/lib/toast"; import { Tick02Icon } from "@/lib/tick-icon"; import { cn } from "@/lib/utils"; @@ -150,6 +152,8 @@ import { RefreshCwIcon, SquareIcon, TerminalIcon, + Volume2Icon, + VolumeXIcon, XIcon, } from "lucide-react"; import { @@ -2118,19 +2122,6 @@ function useImeComposerInputHandlers({ }; } -// Phosphor microphone. Inlined to avoid a new icon dependency. -const MicIcon: FC<{ className?: string }> = ({ className }) => ( - - - -); - // HugeIcons arrow-down-01 (stroke-standard): straight-line chevron. const ArrowDownStandardIcon: FC<{ className?: string }> = ({ className }) => ( { const isRunning = useAuiState(({ thread }) => thread.isRunning); const handleDelete = async () => { - const remoteId = aui.threadListItem().getState().remoteId; const thread = aui.thread(); + // Deleting a message, and for a user prompt its cascaded assistant replies, + // unmounts their only Stop reading control. Stop read-aloud first when the + // spoken message is among those removed. Read speech state at click time and + // guard the call, which throws if playback already ended. + const speakingId = thread.getState().speech?.messageId; + if (speakingId) { + const { messages } = thread.export(); + const target = messages.find(({ message }) => message.id === messageId); + const removed = new Set([messageId]); + if (target?.message.role === "user") { + for (const { parentId, message } of messages) { + if (parentId === messageId && message.role === "assistant") { + removed.add(message.id); + } + } + } + if (removed.has(speakingId)) { + try { + thread.stopSpeaking(); + } catch { + // Playback ended between reading the state and stopping it. + } + } + } + + const remoteId = aui.threadListItem().getState().remoteId; try { await deleteThreadMessage({ thread: { @@ -3883,11 +3899,15 @@ const EditAssistantMessageButton: FC = () => { const AssistantActionBar: FC = () => { const { forkMessage, forkDisabled } = useForkMessageAction(); const [detailsOpen, setDetailsOpen] = useState(false); + const ttsEnabled = useVoiceSettingsStore((s) => s.ttsEnabled); + // hideWhenRunning is thread-level, so a new run would hide this bar and its + // only Stop reading control while read-aloud keeps playing; keep it shown. + const speaking = useAuiState(({ message }) => message.speech != null); return ( <> @@ -3899,6 +3919,28 @@ const AssistantActionBar: FC = () => { + {ttsEnabled && ( + + + + + + + + )} + {/* Not gated on ttsEnabled: turning the setting off while a message + is being read aloud must not remove the only stop control. */} + + + + + + + voice.voiceURI === voiceURI); +} + +// macOS novelty and legacy Eloquence voices that sound robotic and flood the picker. +const LOW_QUALITY_VOICE_NAMES = new Set([ + "albert", + "bad news", + "bahh", + "bells", + "boing", + "bubbles", + "cellos", + "deranged", + "eddy", + "flo", + "fred", + "good news", + "grandma", + "grandpa", + "hysterical", + "jester", + "junior", + "kathy", + "organ", + "princess", + "ralph", + "reed", + "rocko", + "sandy", + "shelley", + "superstar", + "trinoids", + "whisper", + "wobble", + "zarvox", +]); + +function voiceBaseName(voice: SpeechSynthesisVoice): string { + // "Eddy (English (US))" -> "eddy"; "Bad News" -> "bad news" + const name = voice.name.split("(")[0]?.trim().toLowerCase() ?? ""; + return name; +} + +function voiceQualityScore(voice: SpeechSynthesisVoice): number { + const name = voice.name.toLowerCase(); + let score = 0; + if (name.includes("premium")) score += 8; + if (name.includes("enhanced")) score += 7; + if (name.includes("natural") || name.includes("neural")) score += 6; + if (name.includes("siri")) score += 6; + if (name.includes("google")) score += 5; + if (name.includes("microsoft")) score += 4; + if (voice.default) score += 3; + return score; +} + +function langBase(tag: string): string { + return tag.toLowerCase().split(/[-_]/)[0] ?? ""; +} + +const MAX_CURATED_VOICES = 20; + +/** + * Keep the best, most relevant voices: drop low-quality ones, keep English, + * the browser language, and the dictation language, rank by quality hints, + * and cap the list. The selected voice is always kept. + */ +export function curateSystemVoices( + voices: SpeechSynthesisVoice[], + selectedVoiceURI?: string, +): SpeechSynthesisVoice[] { + const { dictationLanguage } = useVoiceSettingsStore.getState(); + const wantedLangs = new Set(["en"]); + if (typeof navigator !== "undefined" && navigator.language) { + wantedLangs.add(langBase(navigator.language)); + } + if (dictationLanguage && dictationLanguage !== "auto") { + wantedLangs.add(langBase(dictationLanguage)); + } + + // WebKit and Linux engines report voices with empty or duplicate voiceURIs; + // drop them so the Radix Select never gets an empty or colliding value. + const seenVoiceURIs = new Set(); + const kept = voices.filter((voice) => { + if (!voice.voiceURI || seenVoiceURIs.has(voice.voiceURI)) return false; + seenVoiceURIs.add(voice.voiceURI); + if (LOW_QUALITY_VOICE_NAMES.has(voiceBaseName(voice))) return false; + return wantedLangs.has(langBase(voice.lang)); + }); + + kept.sort((a, b) => { + const scoreDiff = voiceQualityScore(b) - voiceQualityScore(a); + if (scoreDiff !== 0) return scoreDiff; + return a.name.localeCompare(b.name); + }); + + const curated = kept.slice(0, MAX_CURATED_VOICES); + if ( + selectedVoiceURI && + selectedVoiceURI !== "default" && + !curated.some((voice) => voice.voiceURI === selectedVoiceURI) + ) { + const selected = voices.find( + (voice) => voice.voiceURI === selectedVoiceURI, + ); + if (selected) curated.push(selected); + } + return curated; +} + +/** Build an utterance from the current Voice settings. */ +export function createConfiguredUtterance( + text: string, +): SpeechSynthesisUtterance { + const { ttsVoiceURI, ttsRate, ttsPitch, ttsVolume } = + useVoiceSettingsStore.getState(); + const utterance = new SpeechSynthesisUtterance(text); + const voice = findTtsVoice(ttsVoiceURI); + if (voice) { + utterance.voice = voice; + utterance.lang = voice.lang; + } + utterance.rate = ttsRate; + utterance.pitch = ttsPitch; + utterance.volume = ttsVolume; + return utterance; +} + +/** Generate speech via the loaded TTS audio model; returns a WAV data URL. */ +export async function generateStudioTtsAudio( + text: string, + signal?: AbortSignal, +): Promise { + const response = await authFetch("/api/inference/audio/generate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + messages: [{ role: "user", content: text }], + stream: false, + }), + signal, + }); + if (!response.ok) { + const body = (await response.json().catch(() => null)) as { + detail?: string; + } | null; + const detail = body?.detail ?? `HTTP ${response.status}`; + if (/no model loaded|not an audio model/i.test(detail)) { + throw new Error( + "No TTS model is loaded. Load an audio model (e.g. Orpheus TTS) from the model selector, then try again.", + ); + } + throw new Error(detail); + } + const data = (await response.json()) as { audio?: { data?: string } }; + if (!data.audio?.data) { + throw new Error("The TTS model returned no audio."); + } + return `data:audio/wav;base64,${data.audio.data}`; +} + +function speakWithStudioModel( + text: string, + handleEnd: ( + reason: "finished" | "error" | "cancelled", + error?: unknown, + ) => void, + markRunning: () => void, +): { cancel: () => void } { + const { ttsRate, ttsVolume } = useVoiceSettingsStore.getState(); + const controller = new AbortController(); + let audio: HTMLAudioElement | null = null; + let cancelled = false; + + // Release the element and its multi-MB WAV data URL as soon as playback ends. + const cleanup = () => { + if (audio) { + audio.pause(); + audio.removeAttribute("src"); + audio = null; + } + }; + + void (async () => { + try { + const url = await generateStudioTtsAudio(text, controller.signal); + if (cancelled) return; + audio = new Audio(url); + audio.playbackRate = ttsRate; + audio.volume = ttsVolume; + // Some browsers reset playbackRate to 1 once the source loads; reapply + // it on loadedmetadata so the speed setting reliably takes effect. + audio.addEventListener("loadedmetadata", () => { + if (audio) audio.playbackRate = ttsRate; + }); + audio.addEventListener("ended", () => { + cleanup(); + handleEnd("finished"); + }); + audio.addEventListener("error", () => { + if (cancelled) return; + cleanup(); + handleEnd("error", new Error("Audio playback failed.")); + }); + markRunning(); + await audio.play(); + } catch (error) { + if (cancelled || controller.signal.aborted) return; + cleanup(); + handleEnd("error", error); + } + })(); + + return { + cancel: () => { + cancelled = true; + controller.abort(); + cleanup(); + handleEnd("cancelled"); + }, + }; +} + +/** + * Text-to-speech for assistant messages. Reads Voice settings at speak time. + * Engines: "system" (speechSynthesis) or "studio" (loaded TTS audio model). + */ +export class StudioSpeechSynthesisAdapter implements SpeechSynthesisAdapter { + /** Web Speech synthesis, used by the "system" engine. */ + static systemVoicesSupported(): boolean { + return ( + typeof window !== "undefined" && + "speechSynthesis" in window && + typeof window.SpeechSynthesisUtterance !== "undefined" + ); + } + + // The "studio" engine only needs fetch + Audio playback, so a WebView + // without Web Speech synthesis can still read aloud through the backend. + static isSupported(): boolean { + return ( + StudioSpeechSynthesisAdapter.systemVoicesSupported() || + (typeof window !== "undefined" && typeof window.Audio !== "undefined") + ); + } + + speak(text: string): SpeechSynthesisAdapter.Utterance { + const subscribers = new Set<() => void>(); + + const handleEnd = ( + reason: "finished" | "error" | "cancelled", + error?: unknown, + ) => { + if (res.status.type === "ended") return; + // Surface genuine read-aloud failures; a cancelled/interrupted utterance + // is a normal stop, not an error, and must not toast. + if (reason === "error" && error !== "interrupted" && error !== "canceled") { + toast.error(error instanceof Error ? error.message : "Read aloud failed."); + } + res.status = { type: "ended", reason, error }; + for (const handler of subscribers) handler(); + }; + + let cancelImpl: () => void; + const { ttsEngine } = useVoiceSettingsStore.getState(); + + const res: SpeechSynthesisAdapter.Utterance = { + status: { type: "starting" }, + cancel: () => cancelImpl(), + subscribe: (callback) => { + if (res.status.type === "ended") { + let cancelled = false; + queueMicrotask(() => { + if (!cancelled) callback(); + }); + return () => { + cancelled = true; + }; + } + subscribers.add(callback); + return () => { + subscribers.delete(callback); + }; + }, + }; + + // Fall back to the backend model when the runtime lacks Web Speech + // synthesis (e.g. an audio-only WebView), so read-aloud still works. + if ( + ttsEngine === "studio" || + !StudioSpeechSynthesisAdapter.systemVoicesSupported() + ) { + const session = speakWithStudioModel(text, handleEnd, () => { + if (res.status.type === "ended") return; + // Notify subscribers of the async starting -> running transition; + // the adapter contract drives UI state off these subscribe callbacks. + res.status = { type: "running" }; + for (const handler of subscribers) handler(); + }); + cancelImpl = session.cancel; + return res; + } + + const utterance = createConfiguredUtterance(text); + utterance.addEventListener("end", () => handleEnd("finished")); + utterance.addEventListener("error", (e) => handleEnd("error", e.error)); + + // Chrome silently drops speak() while another utterance is queued from a + // cancelled run; clearing first keeps read-aloud deterministic. + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + res.status = { type: "running" }; + + cancelImpl = () => { + window.speechSynthesis.cancel(); + handleEnd("cancelled"); + }; + return res; + } +} diff --git a/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts b/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts index 146a12bd40..b7a8e904ed 100644 --- a/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts +++ b/studio/frontend/src/features/chat/adapters/studio-web-speech-dictation-adapter.ts @@ -1,10 +1,18 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +import { + applyDictationDictionary, + recordRecentDictation, + resolveDictationLanguage, + useVoiceSettingsStore, +} from "@/features/settings/stores/voice-settings-store"; import type { DictationAdapter } from "@assistant-ui/react"; import { toast } from "sonner"; -const getSpeechRecognitionAPI = (): SpeechRecognitionConstructor | undefined => { +const getSpeechRecognitionAPI = (): + | SpeechRecognitionConstructor + | undefined => { if (typeof window === "undefined") return undefined; return window.SpeechRecognition ?? window.webkitSpeechRecognition; }; @@ -13,23 +21,34 @@ const stopStream = (stream: MediaStream | null) => { stream?.getTracks().forEach((track) => track.stop()); }; -const describeMediaError = (error: unknown): string => { - if (!(error instanceof DOMException)) { - return "Dictation could not access the microphone."; - } - if (error.name === "NotAllowedError") { - return "Microphone access is blocked. Allow microphone access for this Unsloth page, then try again."; - } - if (error.name === "NotFoundError") { - return "No microphone was found for dictation."; - } - if (error.name === "NotReadableError") { - return "The microphone is already in use or unavailable."; - } - return error.message || "Dictation could not access the microphone."; +const mediaErrorName = (error: unknown): unknown => + error && typeof error === "object" && "name" in error + ? (error as { name?: unknown }).name + : undefined; + +/** True for getUserMedia errors meaning the requested device is gone. */ +export const isMissingDeviceError = (error: unknown): boolean => { + const name = mediaErrorName(error); + return name === "OverconstrainedError" || name === "NotFoundError"; }; -const describeSpeechError = (error: string, message?: string): string => { +export const describeMediaError = (error: unknown): string => { + const name = mediaErrorName(error); + if (name === "NotAllowedError" || name === "SecurityError") { + return "Microphone access is blocked. Allow microphone access for this Unsloth page, then try again."; + } + if (name === "NotFoundError" || name === "OverconstrainedError") { + return "No microphone was found for dictation."; + } + if (name === "NotReadableError" || name === "AbortError") { + return "The microphone is already in use or unavailable."; + } + return error instanceof Error && error.message + ? error.message + : "Dictation could not access the microphone."; +}; + +export const describeSpeechError = (error: string, message?: string): string => { if (error === "not-allowed") { return "Speech recognition was blocked by the browser. Check microphone permissions for this Unsloth page."; } @@ -46,7 +65,7 @@ const describeSpeechError = (error: string, message?: string): string => { }; export class StudioWebSpeechDictationAdapter implements DictationAdapter { - private readonly language: string; + private readonly language: string | undefined; private readonly continuous: boolean; private readonly interimResults: boolean; @@ -57,7 +76,8 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { interimResults?: boolean; } = {}, ) { - this.language = options.language ?? navigator.language ?? "en-US"; + // Resolved from Voice settings at listen() time unless overridden. + this.language = options.language; this.continuous = options.continuous ?? true; this.interimResults = options.interimResults ?? true; } @@ -78,13 +98,17 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { } const recognition = new SpeechRecognitionAPI(); - recognition.lang = this.language; + recognition.lang = this.language ?? resolveDictationLanguage(); recognition.continuous = this.continuous; recognition.interimResults = this.interimResults; const speechStartCallbacks = new Set<() => void>(); - const speechEndCallbacks = new Set<(result: DictationAdapter.Result) => void>(); - const speechCallbacks = new Set<(result: DictationAdapter.Result) => void>(); + const speechEndCallbacks = new Set< + (result: DictationAdapter.Result) => void + >(); + const speechCallbacks = new Set< + (result: DictationAdapter.Result) => void + >(); let stream: MediaStream | null = null; let finalTranscript = ""; @@ -147,6 +171,9 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { for (const callback of speechEndCallbacks) { callback({ transcript: finalTranscript }); } + if (reason !== "cancelled") { + recordRecentDictation(finalTranscript); + } finalTranscript = ""; } resolveEnded?.(); @@ -162,14 +189,26 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { recognition.addEventListener("result", (event) => { const speechEvent = event as SpeechRecognitionEvent; - for (let i = speechEvent.resultIndex; i < speechEvent.results.length; i++) { + for ( + let i = speechEvent.resultIndex; + i < speechEvent.results.length; + i++ + ) { const result = speechEvent.results[i]; if (!result) continue; const transcript = result[0]?.transcript ?? ""; if (result.isFinal) { - finalTranscript += transcript; + const corrected = applyDictationDictionary(transcript); + // Join final chunks with a single space so recorded transcripts do + // not merge words when a browser omits leading whitespace. + const trimmed = corrected.trim(); + if (trimmed) { + finalTranscript = finalTranscript + ? `${finalTranscript} ${trimmed}` + : trimmed; + } for (const callback of speechCallbacks) { - callback({ transcript, isFinal: true }); + callback({ transcript: corrected, isFinal: true }); } } else { for (const callback of speechCallbacks) { @@ -189,7 +228,10 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { finish("cancelled"); return; } - const description = describeSpeechError(errorEvent.error, errorEvent.message); + const description = describeSpeechError( + errorEvent.error, + errorEvent.message, + ); console.error("Dictation error:", errorEvent.error, errorEvent.message); toast.error(description); finish("error"); @@ -197,9 +239,30 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { void (async () => { try { - stream = await navigator.mediaDevices.getUserMedia({ - audio: { echoCancellation: true, noiseSuppression: true }, - }); + const { micDeviceId } = useVoiceSettingsStore.getState(); + const baseAudio: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: true, + }; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + micDeviceId && micDeviceId !== "default" + ? { ...baseAudio, deviceId: { exact: micDeviceId } } + : baseAudio, + }); + } catch (error) { + // Saved mic may be unplugged; fall back to the default device. + // Firefox and WebKit throw OverconstrainedError objects that are + // not DOMException instances, so match on the error name. + if (micDeviceId !== "default" && isMissingDeviceError(error)) { + stream = await navigator.mediaDevices.getUserMedia({ + audio: baseAudio, + }); + } else { + throw error; + } + } if (ended) { stopStream(stream); stream = null; @@ -207,13 +270,23 @@ export class StudioWebSpeechDictationAdapter implements DictationAdapter { } const audioTrack = stream.getAudioTracks()[0]; if (!audioTrack || audioTrack.readyState !== "live") { - throw new DOMException("No live microphone track is available.", "NotFoundError"); + throw new DOMException( + "No live microphone track is available.", + "NotFoundError", + ); } try { recognition.start(audioTrack); } catch (error) { - // Older engines expose only start(); retry without the experimental track overload. - console.debug("Dictation start(audioTrack) failed; retrying start().", error); + // Older engines expose only start(); retry without the experimental + // track overload. Recognition then captures from the default device, + // so release the selected-device stream instead of holding it open. + console.debug( + "Dictation start(audioTrack) failed; retrying start().", + error, + ); + stopStream(stream); + stream = null; recognition.start(); } started = true; diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index 634e5ec8b0..20fc979777 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -33,6 +33,7 @@ import { useRef, } from "react"; import { toast } from "sonner"; +import { StudioSpeechSynthesisAdapter } from "./adapters/studio-speech-synthesis-adapter"; import { StudioWebSpeechDictationAdapter } from "./adapters/studio-web-speech-dictation-adapter"; import { ThreadAutosaveHandle, @@ -1033,6 +1034,13 @@ function useStudioRuntimeAdapters( : undefined, [], ); + const speech = useMemo( + () => + StudioSpeechSynthesisAdapter.isSupported() + ? new StudioSpeechSynthesisAdapter() + : undefined, + [], + ); const attachments = useMemo( () => new CompositeAttachmentAdapter([ @@ -1047,8 +1055,8 @@ function useStudioRuntimeAdapters( [], ); const adapters = useMemo( - () => ({ history, dictation, attachments }), - [history, dictation, attachments], + () => ({ history, dictation, speech, attachments }), + [history, dictation, speech, attachments], ); return adapters; diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index 2ed9589461..babff892bb 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -8,6 +8,7 @@ import { } from "@/components/assistant-ui/think-aria-label"; import { Button } from "@/components/ui/button"; import { BulbIcon } from "@/lib/bulb-icon"; +import { MicIcon } from "@/lib/mic-icon"; import { Tick02Icon } from "@/lib/tick-icon"; import { cn } from "@/lib/utils"; import { @@ -22,6 +23,17 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { applyQwenThinkingParams } from "@/features/chat/utils/qwen-params"; +import { + describeMediaError, + describeSpeechError, + isMissingDeviceError, +} from "@/features/chat/adapters/studio-web-speech-dictation-adapter"; +import { + applyDictationDictionary, + recordRecentDictation, + resolveDictationLanguage, + useVoiceSettingsStore, +} from "@/features/settings/stores/voice-settings-store"; import { AUDIO_ACCEPT, MAX_AUDIO_SIZE, fileToBase64 } from "@/lib/audio-utils"; import { isTauri } from "@/lib/api-base"; import { isMultimodalResponse } from "./types/api"; @@ -148,18 +160,6 @@ const ArrowDownStandardIcon: FC<{ className?: string }> = ({ className }) => ( ); -const MicIcon: FC<{ className?: string }> = ({ className }) => ( - - - -); - function isNativeComposing(event: Event) { return "isComposing" in event && (event as InputEvent).isComposing === true; } @@ -214,7 +214,17 @@ function useDictation( const [isDictating, setIsDictating] = useState(false); const recognitionRef = useRef(null); - const start = useCallback(() => { + const streamRef = useRef(null); + const startingRef = useRef(false); + // Guards the getUserMedia await so a mic opened after unmount is released. + const disposedRef = useRef(false); + + const releaseStream = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + }, []); + + const start = useCallback(async () => { const SpeechRecognitionAPI = typeof window !== "undefined" && (window.SpeechRecognition ?? @@ -226,43 +236,136 @@ function useDictation( if (!SpeechRecognitionAPI) { return; } + if (startingRef.current || recognitionRef.current) return; + startingRef.current = true; + + // Open the microphone chosen in Voice settings, matching the main chat + // adapter, so Compare dictation honors the same device selection. + let audioTrack: MediaStreamTrack | undefined; + const { micDeviceId } = useVoiceSettingsStore.getState(); + if (navigator.mediaDevices?.getUserMedia) { + try { + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + micDeviceId && micDeviceId !== "default" + ? { deviceId: { exact: micDeviceId } } + : true, + }); + } catch (error) { + // Saved mic may be unplugged; fall back to the default device. + if (micDeviceId !== "default" && isMissingDeviceError(error)) { + stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + }); + } else { + throw error; + } + } + streamRef.current = stream; + audioTrack = stream.getAudioTracks()[0]; + } catch (error) { + // Permission/security failure: report it and stop instead of silently + // recording from a different default device, matching the main adapter. + startingRef.current = false; + releaseStream(); + setIsDictating(false); + toast.error(describeMediaError(error)); + return; + } + } + + if (disposedRef.current) { + releaseStream(); + startingRef.current = false; + return; + } + const recognition = new SpeechRecognitionAPI() as SpeechRecognition; recognition.continuous = true; recognition.interimResults = true; - recognition.lang = "en-US"; + recognition.lang = resolveDictationLanguage(); + let sessionTranscript = ""; recognition.onresult = (event: SpeechRecognitionEvent) => { - const last = event.resultIndex; - const result = event.results[last]; - if (!result?.isFinal) return; - const transcript = result[0]?.transcript?.trim(); - if (transcript) { + // Iterate every result from resultIndex; a single event can carry more + // than one finalized phrase and dropping the rest loses dictated words. + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i]; + if (!result?.isFinal) continue; + const transcript = applyDictationDictionary( + result[0]?.transcript?.trim() ?? "", + ); + if (!transcript) continue; + sessionTranscript = sessionTranscript + ? `${sessionTranscript} ${transcript}` + : transcript; setText((prev) => (prev ? `${prev} ${transcript}` : transcript)); } }; - recognition.onerror = () => { + recognition.onerror = (event) => { + // Report speech-service failures like the main adapter; aborted is a + // normal stop, not an error. + const errorEvent = event as SpeechRecognitionErrorEvent; + if (errorEvent.error !== "aborted") { + toast.error(describeSpeechError(errorEvent.error, errorEvent.message)); + } setIsDictating(false); }; recognition.onend = () => { - setIsDictating(false); + // A stop()+immediate restart can install a new recognizer before this + // old one ends; only tear down shared refs when we are still current. + if (recognitionRef.current === recognition) { + releaseStream(); + recognitionRef.current = null; + setIsDictating(false); + } + if (sessionTranscript) { + recordRecentDictation(sessionTranscript); + sessionTranscript = ""; + } }; - recognition.start(); + try { + if (audioTrack) { + try { + recognition.start(audioTrack); + } catch { + // No start(track) overload: recognition captures from the default + // device, so release the selected-device stream. + releaseStream(); + recognition.start(); + } + } else { + recognition.start(); + } + } catch { + startingRef.current = false; + releaseStream(); + return; + } recognitionRef.current = recognition; + startingRef.current = false; setIsDictating(true); - }, [setText]); + }, [setText, releaseStream]); const stop = useCallback(() => { if (recognitionRef.current) { recognitionRef.current.stop(); recognitionRef.current = null; } + releaseStream(); setIsDictating(false); - }, []); + }, [releaseStream]); useEffect(() => { + disposedRef.current = false; return () => { + disposedRef.current = true; if (recognitionRef.current) { recognitionRef.current.abort(); } + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; }; }, []); diff --git a/studio/frontend/src/features/settings/settings-dialog.tsx b/studio/frontend/src/features/settings/settings-dialog.tsx index d2f7c092f6..7625449648 100644 --- a/studio/frontend/src/features/settings/settings-dialog.tsx +++ b/studio/frontend/src/features/settings/settings-dialog.tsx @@ -10,6 +10,7 @@ import { } from "@/components/ui/dialog"; import { type TranslationKey, useT } from "@/i18n"; import { cn } from "@/lib/utils"; +import { MicIcon } from "@/lib/mic-icon"; import { Cancel01Icon, CloudIcon, @@ -24,7 +25,14 @@ import { } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { motion, useReducedMotion } from "motion/react"; -import { useDeferredValue, useEffect, useMemo, useRef, useState } from "react"; +import { + type FC, + useDeferredValue, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { SETTINGS_SEARCH_INDEX } from "./settings-search"; import { type SettingsTab, @@ -38,11 +46,14 @@ import { ConnectionsTab } from "./tabs/connections-tab"; import { GeneralTab } from "./tabs/general-tab"; import { ProfileTab } from "./tabs/profile-tab"; import { ResourcesTab } from "./tabs/resources-tab"; +import { VoiceTab } from "./tabs/voice-tab"; interface TabDef { id: SettingsTab; labelKey: TranslationKey; - icon: typeof Settings02Icon; + icon?: typeof Settings02Icon; + /** Plain component icon, for icons shared with chat (not hugeicons). */ + iconComponent?: FC<{ className?: string }>; badgeKey?: TranslationKey; } @@ -76,6 +87,12 @@ const TABS: TabDef[] = [ labelKey: "settings.tabs.connections", icon: CloudIcon, }, + { + id: "voice", + labelKey: "settings.tabs.voice", + iconComponent: MicIcon, + badgeKey: "common.new", + }, { id: "about", labelKey: "settings.tabs.about", icon: HelpCircleIcon }, ]; @@ -91,6 +108,8 @@ function renderTab(tab: SettingsTab) { return ; case "chat": return ; + case "voice": + return ; case "connections": return ; case "api-keys": @@ -189,6 +208,7 @@ export function SettingsDialog() { appearance: null, resources: null, chat: null, + voice: null, connections: null, "api-keys": null, about: null, @@ -279,11 +299,15 @@ export function SettingsDialog() { onClick={() => openResult(tab.id)} className="flex h-[30px] items-center gap-2.5 rounded-full pl-3 pr-2.5 text-[13.5px] font-medium text-muted-foreground transition-colors hover:bg-accent hover:text-accent-foreground" > - + {tab.iconComponent ? ( + + ) : tab.icon ? ( + + ) : null} {tabLabel} {entries.map((entry) => ( @@ -352,11 +376,15 @@ export function SettingsDialog() { } /> )} - + {tab.iconComponent ? ( + + ) : tab.icon ? ( + + ) : null} {t(tab.labelKey)} diff --git a/studio/frontend/src/features/settings/settings-search.ts b/studio/frontend/src/features/settings/settings-search.ts index 36191835e5..786ee11d0a 100644 --- a/studio/frontend/src/features/settings/settings-search.ts +++ b/studio/frontend/src/features/settings/settings-search.ts @@ -98,6 +98,22 @@ export const SETTINGS_SEARCH_INDEX: Record = { "settings.apiKeys.accessTokens", ], connections: [], + voice: [ + "settings.voice.dictation.sectionTitle", + "settings.voice.dictation.microphoneLabel", + "settings.voice.dictation.languageLabel", + "settings.voice.dictation.testLabel", + "settings.voice.dictionary.sectionTitle", + "settings.voice.recents.sectionTitle", + "settings.voice.readAloud.sectionTitle", + "settings.voice.readAloud.buttonLabel", + "settings.voice.readAloud.engineLabel", + "settings.voice.readAloud.voiceLabel", + "settings.voice.readAloud.speedLabel", + "settings.voice.readAloud.pitchLabel", + "settings.voice.readAloud.volumeLabel", + "settings.voice.readAloud.previewLabel", + ], about: [ "settings.about.updates", "settings.about.releaseNotes", diff --git a/studio/frontend/src/features/settings/stores/settings-dialog-store.ts b/studio/frontend/src/features/settings/stores/settings-dialog-store.ts index 234e92b3d0..b9e9c75b14 100644 --- a/studio/frontend/src/features/settings/stores/settings-dialog-store.ts +++ b/studio/frontend/src/features/settings/stores/settings-dialog-store.ts @@ -9,6 +9,7 @@ export type SettingsTab = | "appearance" | "resources" | "chat" + | "voice" | "connections" | "api-keys" | "about"; @@ -63,6 +64,7 @@ function loadInitialTab(): SettingsTab { "appearance", "resources", "chat", + "voice", "connections", "api-keys", "about", diff --git a/studio/frontend/src/features/settings/stores/voice-settings-store.ts b/studio/frontend/src/features/settings/stores/voice-settings-store.ts new file mode 100644 index 0000000000..9e38f4c6ce --- /dev/null +++ b/studio/frontend/src/features/settings/stores/voice-settings-store.ts @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; +import { persist } from "zustand/middleware"; + +// Voice preferences in localStorage. Adapters read them at call time so +// changes apply without reloading the chat runtime. + +export interface RecentDictation { + text: string; + at: number; +} + +const MAX_RECENT_DICTATIONS = 20; +// Cap stored transcript length so a few long dictations cannot bloat the +// persisted blob and trip a synchronous localStorage quota error on save. +const MAX_RECENT_DICTATION_LENGTH = 2000; +const MAX_DICTIONARY_ENTRIES = 100; +const MAX_DICTIONARY_ENTRY_LENGTH = 120; + +export interface VoiceSettingsState { + /** Input device for dictation. "default" = system default microphone. */ + micDeviceId: string; + setMicDeviceId: (value: string) => void; + + /** BCP 47 tag for speech recognition, or "auto" for the browser locale. */ + dictationLanguage: string; + setDictationLanguage: (value: string) => void; + + /** Exact spellings applied to matching transcript words and phrases. */ + dictionary: string[]; + addDictionaryEntry: (value: string) => void; + updateDictionaryEntry: (index: number, value: string) => void; + /** Trim the entry; drop it when it was left empty. Call on input blur. */ + commitDictionaryEntry: (index: number) => void; + removeDictionaryEntry: (index: number) => void; + + /** Final transcripts, newest first, so text can be recovered. */ + recentDictations: RecentDictation[]; + addRecentDictation: (text: string) => void; + clearRecentDictations: () => void; + + /** Show the read-aloud button on assistant responses. */ + ttsEnabled: boolean; + setTtsEnabled: (value: boolean) => void; + + /** "system": speechSynthesis voices. "studio": the loaded TTS audio model. */ + ttsEngine: "system" | "studio"; + setTtsEngine: (value: "system" | "studio") => void; + + /** speechSynthesis voiceURI, or "default" for the system voice. */ + ttsVoiceURI: string; + setTtsVoiceURI: (value: string) => void; + + ttsRate: number; + setTtsRate: (value: number) => void; + ttsPitch: number; + setTtsPitch: (value: number) => void; + ttsVolume: number; + setTtsVolume: (value: number) => void; +} + +export const useVoiceSettingsStore = create()( + persist( + (set) => ({ + micDeviceId: "default", + setMicDeviceId: (micDeviceId) => set({ micDeviceId }), + + dictationLanguage: "auto", + setDictationLanguage: (dictationLanguage) => set({ dictationLanguage }), + + dictionary: [], + addDictionaryEntry: (value) => + set((state) => { + const trimmed = value.trim().slice(0, MAX_DICTIONARY_ENTRY_LENGTH); + if (!trimmed) return state; + if (state.dictionary.length >= MAX_DICTIONARY_ENTRIES) return state; + if ( + state.dictionary.some( + (entry) => entry.toLowerCase() === trimmed.toLowerCase(), + ) + ) { + return state; + } + return { dictionary: [...state.dictionary, trimmed] }; + }), + // Keep the raw value so the input edits freely; commitDictionaryEntry finalizes on blur. + updateDictionaryEntry: (index, value) => + set((state) => { + const dictionary = [...state.dictionary]; + if (index < 0 || index >= dictionary.length) return state; + dictionary[index] = value.slice(0, MAX_DICTIONARY_ENTRY_LENGTH); + return { dictionary }; + }), + commitDictionaryEntry: (index) => + set((state) => { + const dictionary = [...state.dictionary]; + if (index < 0 || index >= dictionary.length) return state; + const trimmed = dictionary[index]?.trim() ?? ""; + if (trimmed) { + dictionary[index] = trimmed; + } else { + dictionary.splice(index, 1); + } + return { dictionary }; + }), + removeDictionaryEntry: (index) => + set((state) => ({ + dictionary: state.dictionary.filter((_, i) => i !== index), + })), + + recentDictations: [], + addRecentDictation: (text) => + set((state) => { + const trimmed = text.trim().slice(0, MAX_RECENT_DICTATION_LENGTH); + if (!trimmed) return state; + return { + recentDictations: [ + { text: trimmed, at: Date.now() }, + ...state.recentDictations, + ].slice(0, MAX_RECENT_DICTATIONS), + }; + }), + clearRecentDictations: () => set({ recentDictations: [] }), + + ttsEnabled: true, + setTtsEnabled: (ttsEnabled) => set({ ttsEnabled }), + + ttsEngine: "system", + setTtsEngine: (ttsEngine) => set({ ttsEngine }), + + ttsVoiceURI: "default", + setTtsVoiceURI: (ttsVoiceURI) => set({ ttsVoiceURI }), + + ttsRate: 1, + setTtsRate: (ttsRate) => set({ ttsRate }), + ttsPitch: 1, + setTtsPitch: (ttsPitch) => set({ ttsPitch }), + ttsVolume: 1, + setTtsVolume: (ttsVolume) => set({ ttsVolume }), + }), + { + name: "unsloth_voice_settings", + merge: (persisted, current) => { + const saved = persisted as Partial | undefined; + return { + ...current, + micDeviceId: asString(saved?.micDeviceId, "default"), + dictationLanguage: asString(saved?.dictationLanguage, "auto"), + dictionary: Array.isArray(saved?.dictionary) + ? saved.dictionary + .filter((v): v is string => typeof v === "string" && !!v.trim()) + .map((v) => v.trim().slice(0, MAX_DICTIONARY_ENTRY_LENGTH)) + .slice(0, MAX_DICTIONARY_ENTRIES) + : [], + recentDictations: Array.isArray(saved?.recentDictations) + ? saved.recentDictations + .filter( + (v): v is RecentDictation => + typeof v?.text === "string" && typeof v?.at === "number", + ) + .slice(0, MAX_RECENT_DICTATIONS) + .map((v) => ({ + text: v.text.slice(0, MAX_RECENT_DICTATION_LENGTH), + at: v.at, + })) + : [], + ttsEnabled: + typeof saved?.ttsEnabled === "boolean" ? saved.ttsEnabled : true, + ttsEngine: saved?.ttsEngine === "studio" ? "studio" : "system", + ttsVoiceURI: asString(saved?.ttsVoiceURI, "default"), + ttsRate: clampNumber(saved?.ttsRate, 0.5, 2, 1), + ttsPitch: clampNumber(saved?.ttsPitch, 0, 2, 1), + ttsVolume: clampNumber(saved?.ttsVolume, 0, 1, 1), + }; + }, + }, + ), +); + +function asString(value: unknown, fallback: string): string { + return typeof value === "string" && value ? value : fallback; +} + +function clampNumber( + value: unknown, + min: number, + max: number, + fallback: number, +): number { + if (typeof value !== "number" || Number.isNaN(value)) return fallback; + return Math.min(max, Math.max(min, value)); +} + +/** Resolve the "auto" language setting to a concrete BCP 47 tag. */ +export function resolveDictationLanguage(setting?: string): string { + const value = setting ?? useVoiceSettingsStore.getState().dictationLanguage; + if (value && value !== "auto") return value; + return typeof navigator !== "undefined" && navigator.language + ? navigator.language + : "en-US"; +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Rewrite dictionary phrases in a transcript to their exact stored form, + * matching case-insensitively on word boundaries ("jane doe" -> "Jane Doe"). + */ +export function applyDictationDictionary( + transcript: string, + dictionary?: string[], +): string { + const entries = dictionary ?? useVoiceSettingsStore.getState().dictionary; + if (!transcript || entries.length === 0) return transcript; + let result = transcript; + for (const entry of entries) { + const trimmed = entry.trim(); + if (!trimmed) continue; + // Whitespace-tolerant pattern so "jane doe" still matches. + const pattern = trimmed.split(/\s+/).map(escapeRegExp).join("\\s+"); + try { + // Capture the leading boundary instead of using a lookbehind, which + // engines that support dictation but not lookbehind (Safari < 16.4) + // cannot compile; the catch below would otherwise skip every entry. + const regex = new RegExp( + `(^|[^\\p{L}\\p{N}])(${pattern})(?![\\p{L}\\p{N}])`, + "giu", + ); + // Re-emit the boundary; callback form avoids $-pattern expansion. + result = result.replace(regex, (_match, prefix) => `${prefix}${trimmed}`); + } catch { + // Skip entries that produce an invalid pattern. + } + } + return result; +} + +/** Record a finished dictation so it can be recovered from settings. */ +export function recordRecentDictation(text: string): void { + useVoiceSettingsStore.getState().addRecentDictation(text); +} diff --git a/studio/frontend/src/features/settings/tabs/general-tab.tsx b/studio/frontend/src/features/settings/tabs/general-tab.tsx index 1b2adf066d..1460c7dea2 100644 --- a/studio/frontend/src/features/settings/tabs/general-tab.tsx +++ b/studio/frontend/src/features/settings/tabs/general-tab.tsx @@ -114,6 +114,8 @@ const PREFS_KEYS: string[] = [ // Update notifications "unsloth_show_llama_update_banner", "unsloth_monitor_overlay", + // Voice settings + "unsloth_voice_settings", ]; // Set by resetAllPrefs so the unmount-commit effect skips writing back the diff --git a/studio/frontend/src/features/settings/tabs/voice-tab.tsx b/studio/frontend/src/features/settings/tabs/voice-tab.tsx new file mode 100644 index 0000000000..4b86105926 --- /dev/null +++ b/studio/frontend/src/features/settings/tabs/voice-tab.tsx @@ -0,0 +1,882 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Slider } from "@/components/ui/slider"; +import { Switch } from "@/components/ui/switch"; +import { + StudioSpeechSynthesisAdapter, + createConfiguredUtterance, + curateSystemVoices, + generateStudioTtsAudio, +} from "@/features/chat/adapters/studio-speech-synthesis-adapter"; +import { + StudioWebSpeechDictationAdapter, + describeSpeechError, + isMissingDeviceError, +} from "@/features/chat/adapters/studio-web-speech-dictation-adapter"; +import { useT } from "@/i18n"; +import { toast } from "@/lib/toast"; +import { MicIcon } from "@/lib/mic-icon"; +import { copyToClipboard } from "@/lib/copy-to-clipboard"; +import { + Copy01Icon, + Delete02Icon, + PlusSignIcon, + VolumeHighIcon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { SquareIcon } from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { SettingsRow } from "../components/settings-row"; +import { SettingsSection } from "../components/settings-section"; +import { + applyDictationDictionary, + recordRecentDictation, + resolveDictationLanguage, + useVoiceSettingsStore, +} from "../stores/voice-settings-store"; + +// Languages offered for browser speech recognition. +const DICTATION_LANGUAGES: { value: string; label: string }[] = [ + { value: "auto", label: "" }, // label rendered via i18n + { value: "en-US", label: "English (US)" }, + { value: "en-GB", label: "English (UK)" }, + { value: "zh-CN", label: "中文 (简体)" }, + { value: "ja-JP", label: "日本語" }, + { value: "ko-KR", label: "한국어" }, + { value: "es-ES", label: "Español" }, + { value: "fr-FR", label: "Français" }, + { value: "de-DE", label: "Deutsch" }, + { value: "it-IT", label: "Italiano" }, + { value: "pt-BR", label: "Português (Brasil)" }, + { value: "ru-RU", label: "Русский" }, + { value: "hi-IN", label: "हिन्दी" }, + { value: "ar-SA", label: "العربية" }, +]; + +const TTS_PREVIEW_TEXT = + "Hello from Unsloth Studio! This is a preview of the selected voice."; + +function useAudioInputDevices() { + const t = useT(); + const [devices, setDevices] = useState([]); + const [hasLabels, setHasLabels] = useState(false); + + const refresh = useCallback(async () => { + if (!navigator.mediaDevices?.enumerateDevices) return; + try { + const all = await navigator.mediaDevices.enumerateDevices(); + const inputs = all.filter((d) => d.kind === "audioinput"); + setDevices(inputs); + setHasLabels(inputs.some((d) => d.label)); + } catch { + // Enumeration can fail in insecure contexts; leave the list empty. + } + }, []); + + useEffect(() => { + void refresh(); + const media = navigator.mediaDevices; + if (!media?.addEventListener) return; + media.addEventListener("devicechange", refresh); + return () => media.removeEventListener("devicechange", refresh); + }, [refresh]); + + // Labels are hidden until mic permission; open a short stream to get them. + const requestAccess = useCallback(async () => { + // Insecure contexts (plain http on a LAN address) have no mediaDevices. + if (!navigator.mediaDevices?.getUserMedia) { + toast.error(t("settings.voice.dictation.micAccessUnsupported")); + return; + } + try { + const stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + }); + stream.getTracks().forEach((track) => track.stop()); + await refresh(); + } catch { + toast.error(t("settings.voice.dictation.micAccessBlocked")); + } + }, [refresh, t]); + + return { devices, hasLabels, requestAccess }; +} + +function useSystemVoices() { + const [voices, setVoices] = useState([]); + + useEffect(() => { + if (typeof window === "undefined" || !window.speechSynthesis) return; + const synth = window.speechSynthesis; + const load = () => setVoices(synth.getVoices()); + load(); + synth.addEventListener?.("voiceschanged", load); + return () => synth.removeEventListener?.("voiceschanged", load); + }, []); + + return voices; +} + +/** Inline mic test: runs speech recognition and shows the live transcript. */ +function DictationTest() { + const t = useT(); + const [testing, setTesting] = useState(false); + const [transcript, setTranscript] = useState(""); + const [interim, setInterim] = useState(""); + const recognitionRef = useRef(null); + const streamRef = useRef(null); + // Guards the getUserMedia await so a mic opened after unmount is released. + const disposedRef = useRef(false); + // Mirrors the transcript state so onend can record it without stale closures. + const transcriptRef = useRef(""); + + // Single cleanup path: the browser can end recognition on its own (silence + // timeout, service disconnect), so onend must release the mic and save the + // transcript, not just the Stop button. + const finalize = useCallback(() => { + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + recognitionRef.current = null; + if (transcriptRef.current) { + recordRecentDictation(transcriptRef.current); + transcriptRef.current = ""; + } + setTesting(false); + setInterim(""); + }, []); + + const stop = useCallback(() => { + const recognition = recognitionRef.current; + if (recognition) { + // onend fires next and runs finalize() + recognition.stop(); + } else { + finalize(); + } + }, [finalize]); + + useEffect(() => { + disposedRef.current = false; + return () => { + disposedRef.current = true; + recognitionRef.current?.abort(); + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + }; + }, []); + + // Set before the getUserMedia await so a double click or a slow + // permission prompt cannot start a second recognizer over the first. + const startingRef = useRef(false); + + const start = useCallback(async () => { + const SpeechRecognitionAPI = + window.SpeechRecognition ?? window.webkitSpeechRecognition; + if (!SpeechRecognitionAPI) return; + if (startingRef.current || recognitionRef.current) return; + startingRef.current = true; + setTranscript(""); + setInterim(""); + transcriptRef.current = ""; + + const { micDeviceId } = useVoiceSettingsStore.getState(); + let audioTrack: MediaStreamTrack | undefined; + try { + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: + micDeviceId && micDeviceId !== "default" + ? { deviceId: { exact: micDeviceId } } + : true, + }); + } catch (error) { + // Saved mic may be unplugged; fall back to the default device. + if (micDeviceId !== "default" && isMissingDeviceError(error)) { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } else { + throw error; + } + } + if (disposedRef.current) { + stream.getTracks().forEach((track) => track.stop()); + startingRef.current = false; + return; + } + streamRef.current = stream; + audioTrack = stream.getAudioTracks()[0]; + } catch { + startingRef.current = false; + toast.error(t("settings.voice.dictation.micOpenFailed")); + return; + } + + const recognition = new SpeechRecognitionAPI(); + recognition.lang = resolveDictationLanguage(); + recognition.continuous = true; + recognition.interimResults = true; + recognition.onresult = (event: SpeechRecognitionEvent) => { + let interimText = ""; + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i]; + const text = result?.[0]?.transcript ?? ""; + if (result?.isFinal) { + const corrected = applyDictationDictionary(text.trim()); + setTranscript((prev) => { + const next = prev ? `${prev} ${corrected}` : corrected; + transcriptRef.current = next; + return next; + }); + } else { + interimText += text; + } + } + setInterim(interimText); + }; + recognition.onerror = (event) => { + // onend follows and runs finalize(); surface non-abort failures here. + const errorEvent = event as SpeechRecognitionErrorEvent; + if (errorEvent.error !== "aborted") { + toast.error(describeSpeechError(errorEvent.error, errorEvent.message)); + } + }; + recognition.onend = () => finalize(); + try { + if (audioTrack) { + try { + recognition.start(audioTrack); + } catch { + // Engine has no start(track) overload: it will capture from the + // default device, so release the selected-device stream. + streamRef.current?.getTracks().forEach((track) => track.stop()); + streamRef.current = null; + recognition.start(); + } + } else { + recognition.start(); + } + } catch { + startingRef.current = false; + finalize(); + return; + } + recognitionRef.current = recognition; + startingRef.current = false; + setTesting(true); + }, [finalize, t]); + + const finishedTest = !testing && transcript; + + return ( +
+ + + + {(testing || transcript) && ( +
+ {transcript || interim ? ( + <> + {transcript} + {interim ? ( + {interim} + ) : null} + + ) : ( + + {testing ? t("settings.voice.dictation.listening") : ""} + + )} + {finishedTest ? ( +
+ {t("settings.voice.dictation.testSaved")} +
+ ) : null} +
+ )} +
+ ); +} + +export function VoiceTab() { + const t = useT(); + const micDeviceId = useVoiceSettingsStore((s) => s.micDeviceId); + const setMicDeviceId = useVoiceSettingsStore((s) => s.setMicDeviceId); + const dictationLanguage = useVoiceSettingsStore((s) => s.dictationLanguage); + const setDictationLanguage = useVoiceSettingsStore( + (s) => s.setDictationLanguage, + ); + const dictionary = useVoiceSettingsStore((s) => s.dictionary); + const addDictionaryEntry = useVoiceSettingsStore((s) => s.addDictionaryEntry); + const updateDictionaryEntry = useVoiceSettingsStore( + (s) => s.updateDictionaryEntry, + ); + const commitDictionaryEntry = useVoiceSettingsStore( + (s) => s.commitDictionaryEntry, + ); + const removeDictionaryEntry = useVoiceSettingsStore( + (s) => s.removeDictionaryEntry, + ); + const recentDictations = useVoiceSettingsStore((s) => s.recentDictations); + const clearRecentDictations = useVoiceSettingsStore( + (s) => s.clearRecentDictations, + ); + const ttsEnabled = useVoiceSettingsStore((s) => s.ttsEnabled); + const setTtsEnabled = useVoiceSettingsStore((s) => s.setTtsEnabled); + const ttsEngine = useVoiceSettingsStore((s) => s.ttsEngine); + const setTtsEngine = useVoiceSettingsStore((s) => s.setTtsEngine); + const ttsVoiceURI = useVoiceSettingsStore((s) => s.ttsVoiceURI); + const setTtsVoiceURI = useVoiceSettingsStore((s) => s.setTtsVoiceURI); + const ttsRate = useVoiceSettingsStore((s) => s.ttsRate); + const setTtsRate = useVoiceSettingsStore((s) => s.setTtsRate); + const ttsPitch = useVoiceSettingsStore((s) => s.ttsPitch); + const setTtsPitch = useVoiceSettingsStore((s) => s.setTtsPitch); + const ttsVolume = useVoiceSettingsStore((s) => s.ttsVolume); + const setTtsVolume = useVoiceSettingsStore((s) => s.setTtsVolume); + + const { devices, hasLabels, requestAccess } = useAudioInputDevices(); + const rawVoices = useSystemVoices(); + const voices = useMemo( + () => curateSystemVoices(rawVoices, ttsVoiceURI), + // dictationLanguage feeds the curation language filter. + [rawVoices, ttsVoiceURI, dictationLanguage], + ); + const [newEntry, setNewEntry] = useState(""); + const [previewing, setPreviewing] = useState(false); + + const dictationSupported = StudioWebSpeechDictationAdapter.isSupported(); + const ttsSupported = StudioSpeechSynthesisAdapter.isSupported(); + const systemTtsSupported = + StudioSpeechSynthesisAdapter.systemVoicesSupported(); + const effectiveTtsEngine = systemTtsSupported ? ttsEngine : "studio"; + + // Keep an item for an unplugged saved mic so the value stays visible. + const knownMic = devices.some((d) => d.deviceId === micDeviceId); + + const handleAddEntry = () => { + const trimmed = newEntry.trim(); + if (!trimmed) return; + addDictionaryEntry(trimmed); + setNewEntry(""); + }; + + const previewAudioRef = useRef(null); + const previewAbortRef = useRef(null); + // Mirrors `previewing` so unmount cleanup can tell whether this tab owns + // the current speechSynthesis utterance; read-aloud shares the global + // synthesizer and must not be cancelled by merely closing settings. + const previewingRef = useRef(false); + // Only a system-voice preview owns the shared speechSynthesis channel; a + // studio (Audio) preview must not cancel an unrelated chat read-aloud. + const ownsSystemPreviewRef = useRef(false); + const markPreviewing = useCallback((value: boolean) => { + previewingRef.current = value; + setPreviewing(value); + }, []); + + const releasePreviewAudio = useCallback(() => { + if (previewAudioRef.current) { + previewAudioRef.current.pause(); + previewAudioRef.current.src = ""; + previewAudioRef.current = null; + } + }, []); + + const stopPreview = useCallback(() => { + if (!previewingRef.current) return; + if (ownsSystemPreviewRef.current) { + window.speechSynthesis?.cancel(); + ownsSystemPreviewRef.current = false; + } + previewAbortRef.current?.abort(); + previewAbortRef.current = null; + releasePreviewAudio(); + markPreviewing(false); + }, [markPreviewing, releasePreviewAudio]); + + const previewTts = async () => { + if (!ttsSupported) return; + // Ref, not state: a double-click before rerender still reads previewing + // as false and would start a second request that orphans the first. + if (previewingRef.current) { + stopPreview(); + return; + } + if (effectiveTtsEngine === "studio") { + const controller = new AbortController(); + previewAbortRef.current = controller; + ownsSystemPreviewRef.current = false; + markPreviewing(true); + try { + const url = await generateStudioTtsAudio( + TTS_PREVIEW_TEXT, + controller.signal, + ); + if (controller.signal.aborted) return; + const audio = new Audio(url); + audio.playbackRate = ttsRate; + audio.volume = ttsVolume; + // Some browsers reset playbackRate to 1 once the source loads; reapply + // it on loadedmetadata so the speed setting reliably takes effect. + audio.addEventListener("loadedmetadata", () => { + audio.playbackRate = ttsRate; + }); + audio.addEventListener("ended", () => { + releasePreviewAudio(); + markPreviewing(false); + }); + audio.addEventListener("error", () => { + releasePreviewAudio(); + markPreviewing(false); + // Surface playback failures like the catch below, instead of just + // resetting the button with no explanation. + toast.error("TTS preview failed"); + }); + previewAudioRef.current = audio; + await audio.play(); + } catch (error) { + if (!controller.signal.aborted) { + toast.error( + error instanceof Error ? error.message : "TTS preview failed", + ); + } + releasePreviewAudio(); + markPreviewing(false); + } + return; + } + if (!StudioSpeechSynthesisAdapter.systemVoicesSupported()) { + toast.error(t("settings.voice.readAloud.notSupported")); + return; + } + const utterance = createConfiguredUtterance(TTS_PREVIEW_TEXT); + utterance.addEventListener("end", () => { + ownsSystemPreviewRef.current = false; + markPreviewing(false); + }); + utterance.addEventListener("error", () => { + ownsSystemPreviewRef.current = false; + markPreviewing(false); + }); + ownsSystemPreviewRef.current = true; + window.speechSynthesis.cancel(); + window.speechSynthesis.speak(utterance); + markPreviewing(true); + }; + + // Stop any preview playback when the tab unmounts. + useEffect(() => stopPreview, [stopPreview]); + + return ( +
+
+

+ {t("settings.voice.title")} +

+

+ {t("settings.voice.description")} +

+
+ + + + {hasLabels ? ( + + ) : ( + + )} + + + + + + + {dictationSupported ? ( + + ) : ( + + )} + + + + {dictionary.map((entry, index) => ( +
+ updateDictionaryEntry(index, e.target.value)} + // Skip the empty-row commit-splice when focus moves to this row's + // Remove button (keyboard Tab), so its index stays valid and its + // activation deletes this row instead of the next one. + onBlur={(e) => { + if ( + (e.relatedTarget as HTMLElement | null)?.dataset.dictRemove === + String(index) + ) { + return; + } + commitDictionaryEntry(index); + }} + className="h-8 flex-1 text-sm" + aria-label={`Dictionary entry ${index + 1}`} + /> + +
+ ))} +
+ setNewEntry(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + handleAddEntry(); + } + }} + placeholder="Jane Doe" + className="h-8 flex-1 text-sm" + aria-label="New dictionary entry" + /> + +
+ + + + {recentDictations.length === 0 ? ( +

+ {t("settings.voice.recents.empty")} +

+ ) : ( + <> + {recentDictations.map((item) => ( +
+
+

+ {item.text} +

+

+ {new Date(item.at).toLocaleString()} +

+
+ +
+ ))} +
+ +
+ + )} + + + + {ttsSupported ? ( + <> + + + + + + + + + {effectiveTtsEngine === "studio" ? ( + + ) : ( + + + + )} + + + v !== undefined && setTtsRate(v)} + className="w-48" + aria-label="Speaking rate" + /> + + + {effectiveTtsEngine === "system" && ( + + v !== undefined && setTtsPitch(v)} + className="w-48" + aria-label="Voice pitch" + /> + + )} + + + v !== undefined && setTtsVolume(v)} + className="w-48" + aria-label="Playback volume" + /> + + + + + + + ) : ( + + )} + +
+ ); +} diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index 3e351c22bc..fe3a6f8542 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -97,10 +97,81 @@ export const en = { appearance: "Appearance", resources: "System", chat: "Chat", + voice: "Voice", connections: "Connections", apiKeys: "API", about: "About", }, + voice: { + title: "Voice", + description: "Microphone, dictation, and read-aloud", + dictation: { + sectionTitle: "Dictation", + microphoneLabel: "Microphone", + microphoneDescription: "Used for dictation", + microphoneFallbackHint: + "Used for dictation. Falls back to the system default if the browser speech engine cannot use this device", + microphoneGrantDescription: "Allow mic access to show device names", + allowMicrophone: "Allow microphone", + micAccessBlocked: + "Microphone access was blocked. Allow microphone access for this Unsloth page, then try again.", + micAccessUnsupported: + "Microphone access is not supported in this browser or context.", + micOpenFailed: + "Could not open the selected microphone. Check permissions or pick another device.", + systemDefault: "System default", + savedMicDisconnected: "Saved microphone (not connected)", + languageLabel: "Dictation language", + languageDescription: "Language to recognize", + languageAuto: "Auto (browser language)", + testLabel: "Test dictation", + testDescription: "Speak to check your mic and settings", + startTest: "Start test", + stopTest: "Stop test", + listening: "Listening…", + testSaved: "Saved to recent dictations", + notSupported: "Not supported in this browser", + }, + dictionary: { + sectionTitle: "Dictation dictionary", + sectionDescription: + "Apply the spelling entered here when dictation recognizes the same words or phrase", + addEntry: "Add entry", + }, + recents: { + sectionTitle: "Recent dictations", + sectionDescription: + "Your recent dictations will appear here so you can recover text", + empty: "No dictations yet", + copied: "Copied to clipboard", + copyFailed: "Could not copy to clipboard", + clear: "Clear recent dictations", + }, + readAloud: { + sectionTitle: "Read aloud", + buttonLabel: "Read aloud button", + buttonDescription: "Show on assistant responses", + engineLabel: "TTS engine", + engineSystemDescription: "Built-in device voices", + engineStudioDescription: "Uses the loaded audio model (e.g. Orpheus)", + engineSystem: "System voices", + engineStudio: "Load TTS model", + modelLabel: "TTS model", + modelDescription: + "Load an audio model from the model selector (e.g. Orpheus TTS)", + voiceLabel: "Voice", + voiceDescription: "Best voices on this device", + speedLabel: "Speed", + pitchLabel: "Pitch", + volumeLabel: "Volume", + previewLabel: "Preview voice", + previewDescription: "Play a short sample", + previewAction: "Preview", + stopAction: "Stop", + ttsLabel: "Text to speech", + notSupported: "Not supported in this browser", + }, + }, general: { title: "General", description: "Global preferences for Unsloth.", @@ -549,7 +620,8 @@ export const en = { codingAgents: "Coding agents", codingAgentsHint: "Launch a coding agent against this server. It uses the loaded model; a local server mints an API key automatically, a remote one includes it in the command.", - codingAgentsSwap: "Swap claude for codex, openclaw, opencode, hermes, or pi.", + codingAgentsSwap: + "Swap claude for codex, openclaw, opencode, hermes, or pi.", codingAgentDetected: "Installed on this machine", codingAgentsDetectedHint: "Detected on this machine: {agents}.", relativeNever: "never", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index 44bb617512..23752b6c26 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -104,6 +104,7 @@ export const ja = { connections: "接続", apiKeys: "API", about: "情報", + voice: "音声", }, general: { title: "一般", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index d689b96428..07832ef1d0 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -103,6 +103,7 @@ export const ptBR = { connections: "Conexões", apiKeys: "API", about: "Sobre", + voice: "Voz", }, general: { title: "Geral", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index fc4e126d72..4c51755244 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -103,6 +103,7 @@ export const zhCN = { connections: "连接", apiKeys: "API", about: "关于", + voice: "语音", }, general: { title: "通用", diff --git a/studio/frontend/src/lib/mic-icon.tsx b/studio/frontend/src/lib/mic-icon.tsx new file mode 100644 index 0000000000..9ff6817845 --- /dev/null +++ b/studio/frontend/src/lib/mic-icon.tsx @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import type { FC } from "react"; + +/** Microphone icon used by the chat composer and the Voice settings tab. */ +export const MicIcon: FC<{ className?: string }> = ({ className }) => ( + + + +);