feat(voice): add audio-reactive TUI animations
This commit is contained in:
Kit Langton
parent
113a117cef
commit
1d772f4896
4 changed files with 237 additions and 86 deletions
1
bun.lock
1
bun.lock
|
|
@ -993,6 +993,7 @@
|
|||
"@opencode-ai/client": "workspace:*",
|
||||
"@opentui/core": "catalog:",
|
||||
"@opentui/solid": "catalog:",
|
||||
"opentui-spinner": "catalog:",
|
||||
"solid-js": "1.9.12",
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
"@opencode-ai/client": "workspace:*",
|
||||
"@opentui/core": "catalog:",
|
||||
"@opentui/solid": "catalog:",
|
||||
"opentui-spinner": "catalog:",
|
||||
"solid-js": "1.9.12"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,7 +51,9 @@ if (!args.server) {
|
|||
const apiKey = process.env["OPENAI_API_KEY"]
|
||||
if (!apiKey) {
|
||||
console.error("OPENAI_API_KEY is required. Run via:")
|
||||
console.error(` 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- bun src/spike.ts ...`)
|
||||
console.error(
|
||||
` 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- bun src/spike.ts ...`,
|
||||
)
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
|
|
@ -226,7 +228,6 @@ const toolHandlers: Record<string, (input: Record<string, unknown>) => Promise<u
|
|||
const sessionID = await requireSession()
|
||||
lastPromptAt = Date.now()
|
||||
await client.session.prompt({ sessionID, text })
|
||||
ui.meta(`[prompt] ${truncate(text, 120)}`)
|
||||
if (input["delivery"] === "background")
|
||||
return { status: "started", sessionID, hint: "Use check_session only when the user asks for status." }
|
||||
await client.session.wait({ sessionID })
|
||||
|
|
@ -461,8 +462,7 @@ const aecBinary = await (async () => {
|
|||
if (args.text || process.platform !== "darwin") return undefined
|
||||
const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url))
|
||||
const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url))
|
||||
if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified)
|
||||
return binary
|
||||
if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary
|
||||
ui.meta("compiling echo-cancellation helper (first run only)...")
|
||||
const { mkdir } = await import("node:fs/promises")
|
||||
await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true })
|
||||
|
|
@ -519,6 +519,24 @@ const soxFormat = ["-q", "-t", "raw", "-r", "24000", "-e", "signed-integer", "-b
|
|||
// PCM16 mono at 24kHz is 48 bytes per millisecond.
|
||||
let playbackEndsAt = 0
|
||||
const assistantSpeaking = () => Date.now() < playbackEndsAt
|
||||
let playbackDoneTimer: ReturnType<typeof setTimeout> | undefined
|
||||
const PLAYBACK_RELEASE_MS = 180
|
||||
|
||||
function pcmLevel(bytes: Buffer) {
|
||||
const samples = Math.floor(bytes.length / 2)
|
||||
if (samples === 0) return 0
|
||||
const stride = Math.max(1, Math.floor(samples / 1200))
|
||||
let energy = 0
|
||||
let count = 0
|
||||
for (let index = 0; index < samples; index += stride) {
|
||||
const sample = bytes.readInt16LE(index * 2) / 32768
|
||||
energy += sample * sample
|
||||
count += 1
|
||||
}
|
||||
const rms = Math.sqrt(energy / count)
|
||||
if (rms < 0.008) return 0
|
||||
return Math.min(1, Math.sqrt((rms - 0.008) / 0.18))
|
||||
}
|
||||
|
||||
let micStarted = false
|
||||
|
||||
|
|
@ -535,8 +553,10 @@ async function startMicrophone() {
|
|||
ui.setStatus({ audio: args.speakers ? "duplex+aec" : "duplex" })
|
||||
ui.meta("mic live — talk any time, even over the assistant")
|
||||
for await (const chunk of audio.stdout as ReadableStream<Uint8Array>) {
|
||||
if (ws?.readyState === WebSocket.OPEN)
|
||||
send({ type: "input_audio_buffer.append", audio: Buffer.from(chunk).toString("base64") })
|
||||
if (ws?.readyState !== WebSocket.OPEN) continue
|
||||
const bytes = Buffer.from(chunk)
|
||||
ui.userAudioLevel(pcmLevel(bytes))
|
||||
send({ type: "input_audio_buffer.append", audio: bytes.toString("base64") })
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -549,7 +569,9 @@ async function startMicrophone() {
|
|||
// Half-duplex: drop mic audio while the assistant is audible (plus a
|
||||
// short tail) so speaker echo can't barge-in against itself.
|
||||
if (!args.duplex && Date.now() < playbackEndsAt + 300) continue
|
||||
send({ type: "input_audio_buffer.append", audio: Buffer.from(chunk).toString("base64") })
|
||||
const bytes = Buffer.from(chunk)
|
||||
ui.userAudioLevel(pcmLevel(bytes))
|
||||
send({ type: "input_audio_buffer.append", audio: bytes.toString("base64") })
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -563,23 +585,40 @@ async function forwardHelperLogs(stream: ReadableStream<Uint8Array>) {
|
|||
}
|
||||
|
||||
function playAudio(base64: string) {
|
||||
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
|
||||
playbackDoneTimer = undefined
|
||||
if (!audio) {
|
||||
player ??= Bun.spawn(["play", ...soxFormat, "-"], { stdin: "pipe", stderr: "ignore" })
|
||||
if (args.debug) void player.exited.then((code) => ui.meta(`[debug] play exited (${code})`))
|
||||
}
|
||||
const stdin = (audio ?? player)!.stdin as import("bun").FileSink
|
||||
const bytes = Buffer.from(base64, "base64")
|
||||
ui.assistantAudio(pcmLevel(bytes), bytes.length / 48)
|
||||
stdin.write(bytes)
|
||||
stdin.flush()
|
||||
playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + bytes.length / 48
|
||||
}
|
||||
|
||||
function finishPlayback() {
|
||||
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
|
||||
playbackDoneTimer = setTimeout(
|
||||
() => {
|
||||
playbackDoneTimer = undefined
|
||||
ui.assistantDone()
|
||||
},
|
||||
Math.max(0, playbackEndsAt - Date.now()) + PLAYBACK_RELEASE_MS,
|
||||
)
|
||||
}
|
||||
|
||||
function flushPlayback() {
|
||||
// The AEC helper flushes its queued speaker audio on SIGUSR1 and keeps running.
|
||||
if (audio) process.kill(audio.pid, "SIGUSR1")
|
||||
player?.kill()
|
||||
player = undefined
|
||||
playbackEndsAt = 0
|
||||
if (playbackDoneTimer) clearTimeout(playbackDoneTimer)
|
||||
playbackDoneTimer = undefined
|
||||
ui.assistantDone()
|
||||
}
|
||||
|
||||
const createResponse = () =>
|
||||
|
|
@ -687,17 +726,22 @@ function onMessage(event: MessageEvent) {
|
|||
ui.assistantDelta(data.delta ?? "")
|
||||
break
|
||||
case "response.done": {
|
||||
ui.assistantDone()
|
||||
if (args.text) ui.assistantDone()
|
||||
else finishPlayback()
|
||||
const calledFunction = data.response?.output?.some((item) => item.type === "function_call") ?? false
|
||||
if (args.text && !calledFunction && inflightTools === 0) shutdown()
|
||||
break
|
||||
}
|
||||
case "input_audio_buffer.speech_started":
|
||||
ui.userSpeaking(true)
|
||||
// Voice barge-in needs full duplex (AEC helper or headphones). In
|
||||
// half-duplex, speech that slips past the mic gate must not kill
|
||||
// playback; keypress is the interrupt.
|
||||
if (fullDuplex) flushPlayback()
|
||||
break
|
||||
case "input_audio_buffer.speech_stopped":
|
||||
ui.userSpeaking(false)
|
||||
break
|
||||
case "input_audio_buffer.committed":
|
||||
// Reserve the user's slot in the conversation now; the transcript
|
||||
// arrives later and must not print after the assistant's reply.
|
||||
|
|
@ -712,9 +756,6 @@ function onMessage(event: MessageEvent) {
|
|||
case "response.output_audio_transcript.delta":
|
||||
ui.assistantDelta(data.delta ?? "")
|
||||
break
|
||||
case "response.output_audio_transcript.done":
|
||||
ui.assistantDone()
|
||||
break
|
||||
case "response.output_item.done":
|
||||
if (data.item?.type === "function_call") void handleFunctionCall(data.item)
|
||||
break
|
||||
|
|
|
|||
|
|
@ -5,8 +5,11 @@
|
|||
// its transcript is filled in when Whisper finishes.
|
||||
import { createCliRenderer, RGBA } from "@opentui/core"
|
||||
import { render, useKeyboard, useTerminalDimensions } from "@opentui/solid"
|
||||
import { registerSpinner } from "opentui-spinner/solid"
|
||||
import { createSignal, For } from "solid-js"
|
||||
|
||||
registerSpinner()
|
||||
|
||||
export type VoiceStatus = {
|
||||
server?: string
|
||||
session?: string
|
||||
|
|
@ -18,8 +21,11 @@ export type VoiceStatus = {
|
|||
|
||||
export type VoiceUI = {
|
||||
meta(text: string): void
|
||||
userSpeaking(active: boolean): void
|
||||
userAudioLevel(level?: number): void
|
||||
userCommitted(itemID: string): void
|
||||
userTranscript(itemID: string, text: string): void
|
||||
assistantAudio(level: number, durationMs: number): void
|
||||
assistantDelta(text: string): void
|
||||
assistantDone(): void
|
||||
toolStart(callID: string, name: string, input: unknown): void
|
||||
|
|
@ -30,7 +36,7 @@ export type VoiceUI = {
|
|||
|
||||
type Message =
|
||||
| { kind: "user"; itemID: string; text?: string }
|
||||
| { kind: "assistant"; text: string; streaming: boolean; reveals: Array<{ offset: number; at: number }> }
|
||||
| { kind: "assistant"; text: string; streaming: boolean }
|
||||
| { kind: "tool"; callID: string; name: string; input: unknown; output?: unknown }
|
||||
| { kind: "meta"; text: string }
|
||||
|
||||
|
|
@ -85,8 +91,11 @@ export function createConsoleUI(): VoiceUI {
|
|||
|
||||
return {
|
||||
meta: (text) => line(dim(` ${text}`)),
|
||||
userSpeaking: () => {},
|
||||
userAudioLevel: () => {},
|
||||
userCommitted: () => {},
|
||||
userTranscript: (_, text) => line(cyan("● you ") + text),
|
||||
assistantAudio: () => {},
|
||||
assistantDelta: (text) => {
|
||||
if (!streaming) {
|
||||
process.stdout.write(green("● assistant "))
|
||||
|
|
@ -127,28 +136,27 @@ const theme = {
|
|||
literal: "#bb9af7",
|
||||
}
|
||||
|
||||
const assistantText = RGBA.fromHex(theme.text)
|
||||
const REVEAL_DURATION_MS = 320
|
||||
const REVEAL_STAGGER_MS = 32
|
||||
const REVEAL_MAX_QUEUE_MS = 160
|
||||
const REVEAL_WORD_LIMIT = 24
|
||||
|
||||
function springOpacity(now: number, start: number) {
|
||||
const progress = Math.max(0, Math.min(1, (now - start) / REVEAL_DURATION_MS))
|
||||
if (progress === 1) return 1
|
||||
const time = progress * 8
|
||||
return 1 - (1 + time) * Math.exp(-time)
|
||||
const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
|
||||
const colors = {
|
||||
text: RGBA.fromHex(theme.text),
|
||||
you: RGBA.fromHex(theme.you),
|
||||
assistant: RGBA.fromHex(theme.assistant),
|
||||
}
|
||||
const AUDIO_FRAME_MS = 33
|
||||
const AUDIO_LEVEL_STALE_MS = 250
|
||||
const meterGlyphs = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"]
|
||||
|
||||
function fade(color: RGBA, opacity: number) {
|
||||
return RGBA.fromValues(color.r, color.g, color.b, color.a * opacity)
|
||||
}
|
||||
|
||||
function revealOffsets(previous: string, delta: string) {
|
||||
const text = previous + delta
|
||||
return Array.from(delta.matchAll(/\S+/g))
|
||||
.map((match) => previous.length + match.index)
|
||||
.filter((offset) => offset === 0 || /\s/.test(text[offset - 1] ?? ""))
|
||||
function meter(level: number, phase: number) {
|
||||
return [0, 1, 2, 3]
|
||||
.map((index) => {
|
||||
const value = Math.max(0, Math.min(1, level * (0.72 + Math.sin(phase + index * 1.4) * 0.28)))
|
||||
return meterGlyphs[Math.round(value * (meterGlyphs.length - 1))]
|
||||
})
|
||||
.join("")
|
||||
}
|
||||
|
||||
// Tiny JSON tokenizer for syntax-highlighted tool results.
|
||||
|
|
@ -176,44 +184,67 @@ function jsonTokens(json: string) {
|
|||
|
||||
// `kind` never changes after creation, so branching once here is safe; the
|
||||
// property reads inside JSX stay reactive through the store proxy.
|
||||
function MessageRow(props: { message: Message; details: boolean; now: () => number }) {
|
||||
function MessageRow(props: { message: Message; details: boolean; assistantLevel: () => number; animate: boolean }) {
|
||||
const message = props.message
|
||||
if (message.kind === "user")
|
||||
return (
|
||||
<text fg={theme.you} marginTop={1}>
|
||||
<b>● you</b> <span style={{ fg: theme.text }}>{message.text ?? "…"}</span>
|
||||
</text>
|
||||
<box
|
||||
flexDirection="column"
|
||||
flexShrink={0}
|
||||
border={["left"]}
|
||||
borderColor={fade(colors.you, 0.38)}
|
||||
backgroundColor={fade(colors.you, 0.035)}
|
||||
paddingLeft={1}
|
||||
paddingRight={1}
|
||||
marginTop={1}
|
||||
>
|
||||
<text fg={fade(colors.you, 0.72)}>
|
||||
<b>you</b>
|
||||
{" "}
|
||||
<span style={{ fg: fade(colors.text, 0.74) }}>{message.text ?? "transcribing"}</span>
|
||||
</text>
|
||||
</box>
|
||||
)
|
||||
if (message.kind === "assistant")
|
||||
if (message.kind === "assistant") {
|
||||
const pulse = () => (!props.animate || !message.streaming ? 0.5 : props.assistantLevel())
|
||||
return (
|
||||
<text fg={theme.assistant} marginTop={1}>
|
||||
<b>● assistant</b>{" "}
|
||||
{message.reveals.length === 0 ? (
|
||||
<span style={{ fg: theme.text }}>{message.text}</span>
|
||||
) : (
|
||||
<>
|
||||
<span style={{ fg: theme.text }}>{message.text.slice(0, message.reveals[0]!.offset)}</span>
|
||||
<For each={message.reveals}>
|
||||
{(reveal, index) => (
|
||||
<span style={{ fg: fade(assistantText, springOpacity(props.now(), reveal.at)) }}>
|
||||
{message.text.slice(reveal.offset, message.reveals[index() + 1]?.offset ?? message.text.length)}
|
||||
</span>
|
||||
)}
|
||||
</For>
|
||||
</>
|
||||
)}
|
||||
</text>
|
||||
<box
|
||||
flexDirection="column"
|
||||
flexShrink={0}
|
||||
border={["left"]}
|
||||
borderColor={fade(colors.assistant, message.streaming ? 0.58 + pulse() * 0.42 : 0.3)}
|
||||
backgroundColor={fade(colors.assistant, message.streaming ? 0.055 + pulse() * 0.11 : 0.025)}
|
||||
paddingLeft={1}
|
||||
paddingRight={1}
|
||||
marginTop={1}
|
||||
>
|
||||
<text fg={fade(colors.assistant, message.streaming ? 0.82 + pulse() * 0.18 : 0.58)}>
|
||||
<b>assistant</b>
|
||||
{" "}
|
||||
<span style={{ fg: fade(colors.text, message.streaming ? 1 : 0.7) }}>{message.text}</span>
|
||||
</text>
|
||||
</box>
|
||||
)
|
||||
}
|
||||
if (message.kind === "tool")
|
||||
return (
|
||||
<box flexDirection="column" paddingLeft={2}>
|
||||
<text fg={theme.muted}>
|
||||
<span style={{ fg: message.output === undefined ? theme.number : theme.assistant }}>
|
||||
{message.output === undefined ? "◌" : "✓"}
|
||||
</span>{" "}
|
||||
<span style={{ fg: theme.key }}>{message.name}</span>{" "}
|
||||
{toolSummary(message.name, message.output)}
|
||||
</text>
|
||||
<box flexDirection="row" gap={1}>
|
||||
{message.output === undefined ? (
|
||||
props.animate ? (
|
||||
<spinner frames={SPINNER_FRAMES} interval={80} color={theme.number} />
|
||||
) : (
|
||||
<text fg={theme.number}>⋯</text>
|
||||
)
|
||||
) : (
|
||||
<text fg={theme.assistant}>✓</text>
|
||||
)}
|
||||
<text fg={theme.muted}>
|
||||
<span style={{ fg: theme.key }}>{message.name}</span>
|
||||
{" "}
|
||||
{toolSummary(message.name, message.output)}
|
||||
</text>
|
||||
</box>
|
||||
{props.details ? (
|
||||
<text fg={theme.muted} paddingLeft={2}>
|
||||
<For each={jsonTokens(displayJson({ input: message.input, output: message.output }))}>
|
||||
|
|
@ -230,6 +261,32 @@ function MessageRow(props: { message: Message; details: boolean; now: () => numb
|
|||
)
|
||||
}
|
||||
|
||||
function UserSpeakingBubble(props: { level: () => number | undefined; now: () => number; animate: boolean }) {
|
||||
const pulse = () => (!props.animate ? 0.5 : (props.level() ?? 0.65))
|
||||
const activity = () => {
|
||||
if (!props.animate) return "voice"
|
||||
return meter(Math.max(0.18, props.level() ?? 0.65), props.now() / (props.level() === undefined ? 220 : 320))
|
||||
}
|
||||
return (
|
||||
<box
|
||||
flexDirection="column"
|
||||
flexShrink={0}
|
||||
border={["left"]}
|
||||
borderColor={fade(colors.you, 0.58 + pulse() * 0.42)}
|
||||
backgroundColor={fade(colors.you, 0.055 + pulse() * 0.11)}
|
||||
paddingLeft={1}
|
||||
paddingRight={1}
|
||||
marginTop={1}
|
||||
>
|
||||
<text fg={fade(colors.you, 0.82 + pulse() * 0.18)}>
|
||||
<b>you</b>
|
||||
{" "}
|
||||
<span style={{ fg: fade(colors.text, 0.8) }}>{activity()}</span>
|
||||
</text>
|
||||
</box>
|
||||
)
|
||||
}
|
||||
|
||||
export async function createVoiceTUI(options: {
|
||||
onInterrupt(): void
|
||||
onExit(): void
|
||||
|
|
@ -242,9 +299,14 @@ export async function createVoiceTUI(options: {
|
|||
details: false,
|
||||
})
|
||||
const [animationFrame, setAnimationFrame] = createSignal(performance.now())
|
||||
const [userActive, setUserActive] = createSignal(false)
|
||||
let animationTimer: ReturnType<typeof setInterval> | undefined
|
||||
let animationEndsAt = 0
|
||||
let revealAt = 0
|
||||
let assistantSegments: Array<{ start: number; end: number; level: number }> = []
|
||||
let assistantScheduledUntil = 0
|
||||
let assistantLevel = 0
|
||||
let userTargetLevel = 0
|
||||
let userLevel = 0
|
||||
let userLevelAt = 0
|
||||
|
||||
const renderer = await createCliRenderer({
|
||||
useMouse: true,
|
||||
|
|
@ -263,29 +325,33 @@ export async function createVoiceTUI(options: {
|
|||
},
|
||||
})
|
||||
|
||||
const animateThrough = (end: number) => {
|
||||
animationEndsAt = Math.max(animationEndsAt, end)
|
||||
const startAnimation = () => {
|
||||
if (options.reducedMotion) return renderer.requestRender()
|
||||
if (animationTimer) return
|
||||
animationTimer = setInterval(() => {
|
||||
const now = performance.now()
|
||||
assistantSegments = assistantSegments.filter((segment) => segment.end > now)
|
||||
const output = assistantSegments.find((segment) => segment.start <= now)
|
||||
const assistantTarget = output?.level ?? 0
|
||||
const userTarget = now - userLevelAt < AUDIO_LEVEL_STALE_MS ? userTargetLevel : 0
|
||||
assistantLevel += (assistantTarget - assistantLevel) * (assistantTarget > assistantLevel ? 0.5 : 0.16)
|
||||
userLevel += (userTarget - userLevel) * (userTarget > userLevel ? 0.5 : 0.18)
|
||||
setAnimationFrame(now)
|
||||
renderer.requestRender()
|
||||
if (now < animationEndsAt) return
|
||||
if (userActive() || assistantSegments.length > 0 || assistantLevel > 0.01 || userLevel > 0.01) return
|
||||
clearInterval(animationTimer)
|
||||
animationTimer = undefined
|
||||
}, 16)
|
||||
}, AUDIO_FRAME_MS)
|
||||
}
|
||||
|
||||
const scheduleReveal = (previous: string, delta: string) => {
|
||||
if (options.reducedMotion) return []
|
||||
const now = performance.now()
|
||||
const offsets = revealOffsets(previous, delta).slice(-REVEAL_WORD_LIMIT)
|
||||
const start = Math.min(Math.max(revealAt, now), now + REVEAL_MAX_QUEUE_MS)
|
||||
const reveals = offsets.map((offset, word) => ({ offset, at: start + word * REVEAL_STAGGER_MS }))
|
||||
if (reveals.length === 0) return reveals
|
||||
revealAt = reveals.at(-1)!.at + REVEAL_STAGGER_MS
|
||||
animateThrough(reveals.at(-1)!.at + REVEAL_DURATION_MS)
|
||||
return reveals
|
||||
const currentAssistantLevel = () => {
|
||||
animationFrame()
|
||||
return assistantLevel
|
||||
}
|
||||
const currentUserLevel = () => {
|
||||
const now = animationFrame()
|
||||
if (now - userLevelAt >= AUDIO_LEVEL_STALE_MS) return undefined
|
||||
return userLevel
|
||||
}
|
||||
|
||||
function App() {
|
||||
|
|
@ -323,8 +389,18 @@ export async function createVoiceTUI(options: {
|
|||
<scrollbox flexGrow={1} stickyScroll stickyStart="bottom" scrollbarOptions={{ visible: false }}>
|
||||
<box flexDirection="column" flexShrink={0}>
|
||||
<For each={state().messages}>
|
||||
{(message) => <MessageRow message={message} details={state().details} now={animationFrame} />}
|
||||
{(message) => (
|
||||
<MessageRow
|
||||
message={message}
|
||||
details={state().details}
|
||||
assistantLevel={currentAssistantLevel}
|
||||
animate={!options.reducedMotion}
|
||||
/>
|
||||
)}
|
||||
</For>
|
||||
{userActive() ? (
|
||||
<UserSpeakingBubble level={currentUserLevel} now={animationFrame} animate={!options.reducedMotion} />
|
||||
) : null}
|
||||
</box>
|
||||
</scrollbox>
|
||||
<box height={1} marginTop={1}>
|
||||
|
|
@ -344,7 +420,25 @@ export async function createVoiceTUI(options: {
|
|||
|
||||
return {
|
||||
meta: (text) => push({ kind: "meta", text }),
|
||||
userCommitted: (itemID) => push({ kind: "user", itemID }),
|
||||
userSpeaking: (active) => {
|
||||
setUserActive(active)
|
||||
if (active) startAnimation()
|
||||
if (!active) userTargetLevel = 0
|
||||
redraw()
|
||||
},
|
||||
userAudioLevel: (level) => {
|
||||
if (level === undefined) {
|
||||
userLevelAt = 0
|
||||
return
|
||||
}
|
||||
userTargetLevel = Math.max(0, Math.min(1, level))
|
||||
userLevelAt = performance.now()
|
||||
if (userActive()) startAnimation()
|
||||
},
|
||||
userCommitted: (itemID) => {
|
||||
setUserActive(false)
|
||||
push({ kind: "user", itemID })
|
||||
},
|
||||
userTranscript: (itemID, text) => {
|
||||
const index = state().messages.findIndex((m) => m.kind === "user" && m.itemID === itemID)
|
||||
if (index === -1) return push({ kind: "user", itemID, text })
|
||||
|
|
@ -356,31 +450,45 @@ export async function createVoiceTUI(options: {
|
|||
}))
|
||||
redraw()
|
||||
},
|
||||
assistantAudio: (level, durationMs) => {
|
||||
if (options.reducedMotion) return redraw()
|
||||
const now = performance.now()
|
||||
const start = Math.max(now, assistantScheduledUntil)
|
||||
const end = start + durationMs
|
||||
assistantSegments.push({ start, end, level: Math.max(0, Math.min(1, level)) })
|
||||
assistantScheduledUntil = end
|
||||
startAnimation()
|
||||
},
|
||||
assistantDelta: (text) => {
|
||||
const index = state().messages.length - 1
|
||||
const last = state().messages[index]
|
||||
if (last?.kind === "assistant" && last.streaming) {
|
||||
const reveals = scheduleReveal(last.text, text)
|
||||
setState((current) => ({
|
||||
...current,
|
||||
messages: current.messages.map((message, i) =>
|
||||
i === index && message.kind === "assistant"
|
||||
? {
|
||||
...message,
|
||||
text: message.text + text,
|
||||
reveals: [...message.reveals, ...reveals].slice(-REVEAL_WORD_LIMIT),
|
||||
}
|
||||
: message,
|
||||
i === index && message.kind === "assistant" ? { ...message, text: message.text + text } : message,
|
||||
),
|
||||
}))
|
||||
redraw()
|
||||
return
|
||||
}
|
||||
push({ kind: "assistant", text, streaming: true, reveals: scheduleReveal("", text) })
|
||||
setState((current) => ({
|
||||
...current,
|
||||
messages: [
|
||||
...current.messages.map((message) =>
|
||||
message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message,
|
||||
),
|
||||
{ kind: "assistant", text, streaming: true },
|
||||
],
|
||||
}))
|
||||
redraw()
|
||||
},
|
||||
assistantDone: () => {
|
||||
const index = state().messages.length - 1
|
||||
if (state().messages[index]?.kind !== "assistant") return
|
||||
assistantSegments = []
|
||||
assistantScheduledUntil = 0
|
||||
assistantLevel = 0
|
||||
const index = state().messages.findLastIndex((message) => message.kind === "assistant" && message.streaming)
|
||||
if (index === -1) return
|
||||
setState((current) => ({
|
||||
...current,
|
||||
messages: current.messages.map((message, i) =>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue