diff --git a/bun.lock b/bun.lock index 8cfc906a62..7e208e42e4 100644 --- a/bun.lock +++ b/bun.lock @@ -993,6 +993,7 @@ "@opencode-ai/client": "workspace:*", "@opentui/core": "catalog:", "@opentui/solid": "catalog:", + "opentui-spinner": "catalog:", "solid-js": "1.9.12", }, "devDependencies": { diff --git a/packages/voice/package.json b/packages/voice/package.json index 4750f30294..bf6ab3db4a 100644 --- a/packages/voice/package.json +++ b/packages/voice/package.json @@ -19,6 +19,7 @@ "@opencode-ai/client": "workspace:*", "@opentui/core": "catalog:", "@opentui/solid": "catalog:", + "opentui-spinner": "catalog:", "solid-js": "1.9.12" } } diff --git a/packages/voice/src/spike.ts b/packages/voice/src/spike.ts index 86efb275ad..084a6dc790 100644 --- a/packages/voice/src/spike.ts +++ b/packages/voice/src/spike.ts @@ -51,7 +51,9 @@ if (!args.server) { const apiKey = process.env["OPENAI_API_KEY"] if (!apiKey) { console.error("OPENAI_API_KEY is required. Run via:") - console.error(` 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- bun src/spike.ts ...`) + console.error( + ` 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- bun src/spike.ts ...`, + ) process.exit(1) } @@ -226,7 +228,6 @@ const toolHandlers: Record) => Promise { if (args.text || process.platform !== "darwin") return undefined const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url)) const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url)) - if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) - return binary + if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary ui.meta("compiling echo-cancellation helper (first run only)...") const { mkdir } = await import("node:fs/promises") await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true }) @@ -519,6 +519,24 @@ const soxFormat = ["-q", "-t", "raw", "-r", "24000", "-e", "signed-integer", "-b // PCM16 mono at 24kHz is 48 bytes per millisecond. let playbackEndsAt = 0 const assistantSpeaking = () => Date.now() < playbackEndsAt +let playbackDoneTimer: ReturnType | undefined +const PLAYBACK_RELEASE_MS = 180 + +function pcmLevel(bytes: Buffer) { + const samples = Math.floor(bytes.length / 2) + if (samples === 0) return 0 + const stride = Math.max(1, Math.floor(samples / 1200)) + let energy = 0 + let count = 0 + for (let index = 0; index < samples; index += stride) { + const sample = bytes.readInt16LE(index * 2) / 32768 + energy += sample * sample + count += 1 + } + const rms = Math.sqrt(energy / count) + if (rms < 0.008) return 0 + return Math.min(1, Math.sqrt((rms - 0.008) / 0.18)) +} let micStarted = false @@ -535,8 +553,10 @@ async function startMicrophone() { ui.setStatus({ audio: args.speakers ? "duplex+aec" : "duplex" }) ui.meta("mic live — talk any time, even over the assistant") for await (const chunk of audio.stdout as ReadableStream) { - if (ws?.readyState === WebSocket.OPEN) - send({ type: "input_audio_buffer.append", audio: Buffer.from(chunk).toString("base64") }) + if (ws?.readyState !== WebSocket.OPEN) continue + const bytes = Buffer.from(chunk) + ui.userAudioLevel(pcmLevel(bytes)) + send({ type: "input_audio_buffer.append", audio: bytes.toString("base64") }) } return } @@ -549,7 +569,9 @@ async function startMicrophone() { // Half-duplex: drop mic audio while the assistant is audible (plus a // short tail) so speaker echo can't barge-in against itself. if (!args.duplex && Date.now() < playbackEndsAt + 300) continue - send({ type: "input_audio_buffer.append", audio: Buffer.from(chunk).toString("base64") }) + const bytes = Buffer.from(chunk) + ui.userAudioLevel(pcmLevel(bytes)) + send({ type: "input_audio_buffer.append", audio: bytes.toString("base64") }) } } @@ -563,23 +585,40 @@ async function forwardHelperLogs(stream: ReadableStream) { } function playAudio(base64: string) { + if (playbackDoneTimer) clearTimeout(playbackDoneTimer) + playbackDoneTimer = undefined if (!audio) { player ??= Bun.spawn(["play", ...soxFormat, "-"], { stdin: "pipe", stderr: "ignore" }) if (args.debug) void player.exited.then((code) => ui.meta(`[debug] play exited (${code})`)) } const stdin = (audio ?? player)!.stdin as import("bun").FileSink const bytes = Buffer.from(base64, "base64") + ui.assistantAudio(pcmLevel(bytes), bytes.length / 48) stdin.write(bytes) stdin.flush() playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + bytes.length / 48 } +function finishPlayback() { + if (playbackDoneTimer) clearTimeout(playbackDoneTimer) + playbackDoneTimer = setTimeout( + () => { + playbackDoneTimer = undefined + ui.assistantDone() + }, + Math.max(0, playbackEndsAt - Date.now()) + PLAYBACK_RELEASE_MS, + ) +} + function flushPlayback() { // The AEC helper flushes its queued speaker audio on SIGUSR1 and keeps running. if (audio) process.kill(audio.pid, "SIGUSR1") player?.kill() player = undefined playbackEndsAt = 0 + if (playbackDoneTimer) clearTimeout(playbackDoneTimer) + playbackDoneTimer = undefined + ui.assistantDone() } const createResponse = () => @@ -687,17 +726,22 @@ function onMessage(event: MessageEvent) { ui.assistantDelta(data.delta ?? "") break case "response.done": { - ui.assistantDone() + if (args.text) ui.assistantDone() + else finishPlayback() const calledFunction = data.response?.output?.some((item) => item.type === "function_call") ?? false if (args.text && !calledFunction && inflightTools === 0) shutdown() break } case "input_audio_buffer.speech_started": + ui.userSpeaking(true) // Voice barge-in needs full duplex (AEC helper or headphones). In // half-duplex, speech that slips past the mic gate must not kill // playback; keypress is the interrupt. if (fullDuplex) flushPlayback() break + case "input_audio_buffer.speech_stopped": + ui.userSpeaking(false) + break case "input_audio_buffer.committed": // Reserve the user's slot in the conversation now; the transcript // arrives later and must not print after the assistant's reply. @@ -712,9 +756,6 @@ function onMessage(event: MessageEvent) { case "response.output_audio_transcript.delta": ui.assistantDelta(data.delta ?? "") break - case "response.output_audio_transcript.done": - ui.assistantDone() - break case "response.output_item.done": if (data.item?.type === "function_call") void handleFunctionCall(data.item) break diff --git a/packages/voice/src/ui.tsx b/packages/voice/src/ui.tsx index 97a8b2b376..8f39444ae5 100644 --- a/packages/voice/src/ui.tsx +++ b/packages/voice/src/ui.tsx @@ -5,8 +5,11 @@ // its transcript is filled in when Whisper finishes. import { createCliRenderer, RGBA } from "@opentui/core" import { render, useKeyboard, useTerminalDimensions } from "@opentui/solid" +import { registerSpinner } from "opentui-spinner/solid" import { createSignal, For } from "solid-js" +registerSpinner() + export type VoiceStatus = { server?: string session?: string @@ -18,8 +21,11 @@ export type VoiceStatus = { export type VoiceUI = { meta(text: string): void + userSpeaking(active: boolean): void + userAudioLevel(level?: number): void userCommitted(itemID: string): void userTranscript(itemID: string, text: string): void + assistantAudio(level: number, durationMs: number): void assistantDelta(text: string): void assistantDone(): void toolStart(callID: string, name: string, input: unknown): void @@ -30,7 +36,7 @@ export type VoiceUI = { type Message = | { kind: "user"; itemID: string; text?: string } - | { kind: "assistant"; text: string; streaming: boolean; reveals: Array<{ offset: number; at: number }> } + | { kind: "assistant"; text: string; streaming: boolean } | { kind: "tool"; callID: string; name: string; input: unknown; output?: unknown } | { kind: "meta"; text: string } @@ -85,8 +91,11 @@ export function createConsoleUI(): VoiceUI { return { meta: (text) => line(dim(` ${text}`)), + userSpeaking: () => {}, + userAudioLevel: () => {}, userCommitted: () => {}, userTranscript: (_, text) => line(cyan("● you ") + text), + assistantAudio: () => {}, assistantDelta: (text) => { if (!streaming) { process.stdout.write(green("● assistant ")) @@ -127,28 +136,27 @@ const theme = { literal: "#bb9af7", } -const assistantText = RGBA.fromHex(theme.text) -const REVEAL_DURATION_MS = 320 -const REVEAL_STAGGER_MS = 32 -const REVEAL_MAX_QUEUE_MS = 160 -const REVEAL_WORD_LIMIT = 24 - -function springOpacity(now: number, start: number) { - const progress = Math.max(0, Math.min(1, (now - start) / REVEAL_DURATION_MS)) - if (progress === 1) return 1 - const time = progress * 8 - return 1 - (1 + time) * Math.exp(-time) +const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] +const colors = { + text: RGBA.fromHex(theme.text), + you: RGBA.fromHex(theme.you), + assistant: RGBA.fromHex(theme.assistant), } +const AUDIO_FRAME_MS = 33 +const AUDIO_LEVEL_STALE_MS = 250 +const meterGlyphs = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"] function fade(color: RGBA, opacity: number) { return RGBA.fromValues(color.r, color.g, color.b, color.a * opacity) } -function revealOffsets(previous: string, delta: string) { - const text = previous + delta - return Array.from(delta.matchAll(/\S+/g)) - .map((match) => previous.length + match.index) - .filter((offset) => offset === 0 || /\s/.test(text[offset - 1] ?? "")) +function meter(level: number, phase: number) { + return [0, 1, 2, 3] + .map((index) => { + const value = Math.max(0, Math.min(1, level * (0.72 + Math.sin(phase + index * 1.4) * 0.28))) + return meterGlyphs[Math.round(value * (meterGlyphs.length - 1))] + }) + .join("") } // Tiny JSON tokenizer for syntax-highlighted tool results. @@ -176,44 +184,67 @@ function jsonTokens(json: string) { // `kind` never changes after creation, so branching once here is safe; the // property reads inside JSX stay reactive through the store proxy. -function MessageRow(props: { message: Message; details: boolean; now: () => number }) { +function MessageRow(props: { message: Message; details: boolean; assistantLevel: () => number; animate: boolean }) { const message = props.message if (message.kind === "user") return ( - - ● you {message.text ?? "…"} - + + + you + {" "} + {message.text ?? "transcribing"} + + ) - if (message.kind === "assistant") + if (message.kind === "assistant") { + const pulse = () => (!props.animate || !message.streaming ? 0.5 : props.assistantLevel()) return ( - - ● assistant{" "} - {message.reveals.length === 0 ? ( - {message.text} - ) : ( - <> - {message.text.slice(0, message.reveals[0]!.offset)} - - {(reveal, index) => ( - - {message.text.slice(reveal.offset, message.reveals[index() + 1]?.offset ?? message.text.length)} - - )} - - - )} - + + + assistant + {" "} + {message.text} + + ) + } if (message.kind === "tool") return ( - - - {message.output === undefined ? "◌" : "✓"} - {" "} - {message.name}{" "} - {toolSummary(message.name, message.output)} - + + {message.output === undefined ? ( + props.animate ? ( + + ) : ( + ⋯ + ) + ) : ( + ✓ + )} + + {message.name} + {" "} + {toolSummary(message.name, message.output)} + + {props.details ? ( @@ -230,6 +261,32 @@ function MessageRow(props: { message: Message; details: boolean; now: () => numb ) } +function UserSpeakingBubble(props: { level: () => number | undefined; now: () => number; animate: boolean }) { + const pulse = () => (!props.animate ? 0.5 : (props.level() ?? 0.65)) + const activity = () => { + if (!props.animate) return "voice" + return meter(Math.max(0.18, props.level() ?? 0.65), props.now() / (props.level() === undefined ? 220 : 320)) + } + return ( + + + you + {" "} + {activity()} + + + ) +} + export async function createVoiceTUI(options: { onInterrupt(): void onExit(): void @@ -242,9 +299,14 @@ export async function createVoiceTUI(options: { details: false, }) const [animationFrame, setAnimationFrame] = createSignal(performance.now()) + const [userActive, setUserActive] = createSignal(false) let animationTimer: ReturnType | undefined - let animationEndsAt = 0 - let revealAt = 0 + let assistantSegments: Array<{ start: number; end: number; level: number }> = [] + let assistantScheduledUntil = 0 + let assistantLevel = 0 + let userTargetLevel = 0 + let userLevel = 0 + let userLevelAt = 0 const renderer = await createCliRenderer({ useMouse: true, @@ -263,29 +325,33 @@ export async function createVoiceTUI(options: { }, }) - const animateThrough = (end: number) => { - animationEndsAt = Math.max(animationEndsAt, end) + const startAnimation = () => { + if (options.reducedMotion) return renderer.requestRender() if (animationTimer) return animationTimer = setInterval(() => { const now = performance.now() + assistantSegments = assistantSegments.filter((segment) => segment.end > now) + const output = assistantSegments.find((segment) => segment.start <= now) + const assistantTarget = output?.level ?? 0 + const userTarget = now - userLevelAt < AUDIO_LEVEL_STALE_MS ? userTargetLevel : 0 + assistantLevel += (assistantTarget - assistantLevel) * (assistantTarget > assistantLevel ? 0.5 : 0.16) + userLevel += (userTarget - userLevel) * (userTarget > userLevel ? 0.5 : 0.18) setAnimationFrame(now) renderer.requestRender() - if (now < animationEndsAt) return + if (userActive() || assistantSegments.length > 0 || assistantLevel > 0.01 || userLevel > 0.01) return clearInterval(animationTimer) animationTimer = undefined - }, 16) + }, AUDIO_FRAME_MS) } - const scheduleReveal = (previous: string, delta: string) => { - if (options.reducedMotion) return [] - const now = performance.now() - const offsets = revealOffsets(previous, delta).slice(-REVEAL_WORD_LIMIT) - const start = Math.min(Math.max(revealAt, now), now + REVEAL_MAX_QUEUE_MS) - const reveals = offsets.map((offset, word) => ({ offset, at: start + word * REVEAL_STAGGER_MS })) - if (reveals.length === 0) return reveals - revealAt = reveals.at(-1)!.at + REVEAL_STAGGER_MS - animateThrough(reveals.at(-1)!.at + REVEAL_DURATION_MS) - return reveals + const currentAssistantLevel = () => { + animationFrame() + return assistantLevel + } + const currentUserLevel = () => { + const now = animationFrame() + if (now - userLevelAt >= AUDIO_LEVEL_STALE_MS) return undefined + return userLevel } function App() { @@ -323,8 +389,18 @@ export async function createVoiceTUI(options: { - {(message) => } + {(message) => ( + + )} + {userActive() ? ( + + ) : null} @@ -344,7 +420,25 @@ export async function createVoiceTUI(options: { return { meta: (text) => push({ kind: "meta", text }), - userCommitted: (itemID) => push({ kind: "user", itemID }), + userSpeaking: (active) => { + setUserActive(active) + if (active) startAnimation() + if (!active) userTargetLevel = 0 + redraw() + }, + userAudioLevel: (level) => { + if (level === undefined) { + userLevelAt = 0 + return + } + userTargetLevel = Math.max(0, Math.min(1, level)) + userLevelAt = performance.now() + if (userActive()) startAnimation() + }, + userCommitted: (itemID) => { + setUserActive(false) + push({ kind: "user", itemID }) + }, userTranscript: (itemID, text) => { const index = state().messages.findIndex((m) => m.kind === "user" && m.itemID === itemID) if (index === -1) return push({ kind: "user", itemID, text }) @@ -356,31 +450,45 @@ export async function createVoiceTUI(options: { })) redraw() }, + assistantAudio: (level, durationMs) => { + if (options.reducedMotion) return redraw() + const now = performance.now() + const start = Math.max(now, assistantScheduledUntil) + const end = start + durationMs + assistantSegments.push({ start, end, level: Math.max(0, Math.min(1, level)) }) + assistantScheduledUntil = end + startAnimation() + }, assistantDelta: (text) => { const index = state().messages.length - 1 const last = state().messages[index] if (last?.kind === "assistant" && last.streaming) { - const reveals = scheduleReveal(last.text, text) setState((current) => ({ ...current, messages: current.messages.map((message, i) => - i === index && message.kind === "assistant" - ? { - ...message, - text: message.text + text, - reveals: [...message.reveals, ...reveals].slice(-REVEAL_WORD_LIMIT), - } - : message, + i === index && message.kind === "assistant" ? { ...message, text: message.text + text } : message, ), })) redraw() return } - push({ kind: "assistant", text, streaming: true, reveals: scheduleReveal("", text) }) + setState((current) => ({ + ...current, + messages: [ + ...current.messages.map((message) => + message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message, + ), + { kind: "assistant", text, streaming: true }, + ], + })) + redraw() }, assistantDone: () => { - const index = state().messages.length - 1 - if (state().messages[index]?.kind !== "assistant") return + assistantSegments = [] + assistantScheduledUntil = 0 + assistantLevel = 0 + const index = state().messages.findLastIndex((message) => message.kind === "assistant" && message.streaming) + if (index === -1) return setState((current) => ({ ...current, messages: current.messages.map((message, i) =>