diff --git a/bun.lock b/bun.lock index 7e208e42e4..8484dbb13a 100644 --- a/bun.lock +++ b/bun.lock @@ -991,10 +991,12 @@ "version": "0.0.0", "dependencies": { "@opencode-ai/client": "workspace:*", + "@opencode-ai/plugin": "workspace:*", "@opentui/core": "catalog:", "@opentui/solid": "catalog:", + "effect": "catalog:", "opentui-spinner": "catalog:", - "solid-js": "1.9.12", + "solid-js": "catalog:", }, "devDependencies": { "@tsconfig/bun": "catalog:", @@ -6312,8 +6314,6 @@ "@opencode-ai/updates/wrangler": ["wrangler@4.110.0", "", { "dependencies": { "@cloudflare/kv-asset-handler": "0.5.0", "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.28.1", "miniflare": "4.20260708.1", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", "workerd": "1.20260708.1" }, "optionalDependencies": { "fsevents": "2.3.3" }, "peerDependencies": { "@cloudflare/workers-types": "^5.20260708.1" }, "optionalPeers": ["@cloudflare/workers-types"], "bin": { "wrangler": "bin/wrangler.js", "wrangler2": "bin/wrangler.js", "cf-wrangler": "bin/cf-wrangler.js" } }, "sha512-xZeXKYi7hxQRF5anL+v77RkufJNpF9f3Eqeyqq2QBsETpLZgh0Agj0jJ6JPtkbgn6ukZdh8OK5egsGPWIditgg=="], - "@opencode-ai/voice/solid-js": ["solid-js@1.9.12", "", { "dependencies": { "csstype": "^3.1.0", "seroval": "~1.5.0", "seroval-plugins": "~1.5.0" } }, "sha512-QzKaSJq2/iDrWR1As6MHZQ8fQkdOBf8GReYb7L5iKwMGceg7HxDcaOHk0at66tNgn9U2U7dXo8ZZpLIAmGMzgw=="], - "@opencode-ai/web/@shikijs/transformers": ["@shikijs/transformers@3.20.0", "", { "dependencies": { "@shikijs/core": "3.20.0", "@shikijs/types": "3.20.0" } }, "sha512-PrHHMRr3Q5W1qB/42kJW6laqFyWdhrPF2hNR9qjOm1xcSiAO3hAHo7HaVyHE6pMyevmy3i51O8kuGGXC78uK3g=="], "@opencode-ai/www/@astrojs/cloudflare": ["@astrojs/cloudflare@14.1.4", "", { "dependencies": { "@astrojs/internal-helpers": "0.10.1", "@astrojs/underscore-redirects": "1.0.3", "@cloudflare/vite-plugin": "^1.39.0", "piccolore": "^0.1.3", "vite": "^8.0.13" }, "peerDependencies": { "astro": "^7.0.0", "wrangler": "^4.83.0" } }, "sha512-Zyo1E/5/dmegmKODbwUzOd67euNd6oKcbllhAwe3uFFprA7mIFNkpF6yBBa78vIkpMNuU13MkBJHCLd+yJSuEA=="], @@ -7350,10 +7350,6 @@ "@opencode-ai/updates/wrangler/workerd": ["workerd@1.20260708.1", "", { "optionalDependencies": { "@cloudflare/workerd-darwin-64": "1.20260708.1", "@cloudflare/workerd-darwin-arm64": "1.20260708.1", "@cloudflare/workerd-linux-64": "1.20260708.1", "@cloudflare/workerd-linux-arm64": "1.20260708.1", "@cloudflare/workerd-windows-64": "1.20260708.1" }, "bin": { "workerd": "bin/workerd" } }, "sha512-WAK+Kt/VVCSldH2qSr8lx46XCJ4Q+bdlHNaFqUtOHthBEIB8C1N8HVW+VOLrxDoTCk0NGNv0zajnBeQK4JOB9w=="], - "@opencode-ai/voice/solid-js/seroval": ["seroval@1.5.5", "", {}, "sha512-bSjOuPcwPKLSJNhr9+bZxA20nQxVle5J5MNsYRVE6cIg7KpRLXGupymePavu0jrxlPiPsr4xGZSB8yUY2sH2sw=="], - - "@opencode-ai/voice/solid-js/seroval-plugins": ["seroval-plugins@1.5.5", "", { "peerDependencies": { "seroval": "^1.0" } }, "sha512-+BDhqYM6CEn3x09v44dpa9p6974FuUB2dxk+Ctn04k0cO1Zt6QODTXfmEZK0eBaTe/fJBvP4NMGuNJ+R8T+QMg=="], - "@opencode-ai/web/@shikijs/transformers/@shikijs/core": ["@shikijs/core@3.20.0", "", { "dependencies": { "@shikijs/types": "3.20.0", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4", "hast-util-to-html": "^9.0.5" } }, "sha512-f2ED7HYV4JEk827mtMDwe/yQ25pRiXZmtHjWF8uzZKuKiEsJR7Ce1nuQ+HhV9FzDcbIo4ObBCD9GPTzNuy9S1g=="], "@opencode-ai/web/@shikijs/transformers/@shikijs/types": ["@shikijs/types@3.20.0", "", { "dependencies": { "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-lhYAATn10nkZcBQ0BlzSbJA3wcmL5MXUUF8d2Zzon6saZDlToKaiRX60n2+ZaHJCmXEcZRWNzn+k9vplr8Jhsw=="], diff --git a/packages/voice/package.json b/packages/voice/package.json index bf6ab3db4a..6b550f741d 100644 --- a/packages/voice/package.json +++ b/packages/voice/package.json @@ -3,10 +3,12 @@ "name": "@opencode-ai/voice", "version": "0.0.0", "private": true, - "description": "Prototype voice control for OpenCode via the OpenAI Realtime API", + "description": "Prototype voice control for OpenCode via the OpenAI Realtime and Live APIs", "type": "module", "scripts": { + "bench:audio": "bun scripts/bench-speaker-queue.ts", "spike": "bun run --no-orphans --conditions=browser src/spike.ts", + "test": "bun test", "typecheck": "tsgo --noEmit" }, "devDependencies": { @@ -17,9 +19,11 @@ }, "dependencies": { "@opencode-ai/client": "workspace:*", + "@opencode-ai/plugin": "workspace:*", "@opentui/core": "catalog:", "@opentui/solid": "catalog:", + "effect": "catalog:", "opentui-spinner": "catalog:", - "solid-js": "1.9.12" + "solid-js": "catalog:" } } diff --git a/packages/voice/perf/speaker-queue.md b/packages/voice/perf/speaker-queue.md new file mode 100644 index 0000000000..2b04123912 --- /dev/null +++ b/packages/voice/perf/speaker-queue.md @@ -0,0 +1,38 @@ +# Speaker Queue Performance + +## Goal + +Reduce render-path latency and variance that can produce audible output glitches. + +## Benchmark + +Run `bun run bench:audio` from `packages/voice`. + +The benchmark compiles the production `SpeakerQueue`, performs one warmup run and seven measured runs, and reports callback latency for a 512-frame pop followed by a same-sized producer push. The queue starts with one second of 24 kHz PCM audio. + +## Metrics + +- Primary: `speaker_queue_p99_ns` +- Secondary: `speaker_queue_median_ns`, `speaker_queue_worst_ns` + +## Scope + +- `src/duplex-audio.swift` +- `scripts/bench-speaker-queue.ts` + +## Experiments + +### Baseline: allocating `Data` queue + +- Median: 3,750 ns +- p99: 5,833 ns +- Worst: 92,209 ns median across runs; one run reached 7,497,334 ns +- Decision: baseline + +### Experiment 1: preallocated circular queue + +- Hypothesis: rendering directly from preallocated storage will reduce callback tail latency by avoiding an array allocation and `Data.removeFirst()` on every callback. +- Median: 1,417 ns, down 62.2% +- p99: 1,917 ns, down 67.1% +- Worst: 32,709 ns median across runs, down 64.5% +- Decision: keep diff --git a/packages/voice/scripts/bench-speaker-queue.ts b/packages/voice/scripts/bench-speaker-queue.ts new file mode 100644 index 0000000000..1c73ed1f42 --- /dev/null +++ b/packages/voice/scripts/bench-speaker-queue.ts @@ -0,0 +1,43 @@ +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" + +const directory = await mkdtemp(join(tmpdir(), "opencode-voice-bench-")) +const binary = join(directory, "speaker-queue") +const source = Bun.fileURLToPath(new URL("../src/duplex-audio.swift", import.meta.url)) +const compile = Bun.spawn(["swiftc", "-O", "-D", "QUEUE_BENCHMARK", source, "-o", binary], { + stdout: "ignore", + stderr: "pipe", +}) +if ((await compile.exited) !== 0) { + console.error(await new Response(compile.stderr).text()) + await rm(directory, { recursive: true }) + process.exit(1) +} + +const runs: Array> = [] +for (let index = 0; index < 8; index++) { + const process = Bun.spawn([binary], { stdout: "pipe", stderr: "inherit" }) + const output = await new Response(process.stdout).text() + if ((await process.exited) !== 0) throw new Error(`benchmark run ${index + 1} failed`) + if (index === 0) continue + runs.push( + Object.fromEntries( + output + .split("\n") + .filter((line) => line.startsWith("METRIC ")) + .map((line) => { + const [name, value] = line.slice(7).split("=") + return [name!, Number(value)] + }), + ), + ) +} +await rm(directory, { recursive: true }) + +for (const name of ["speaker_queue_median_ns", "speaker_queue_p99_ns", "speaker_queue_worst_ns"]) { + const values = runs.map((run) => run[name]!).sort((a, b) => a - b) + const median = values[Math.floor(values.length / 2)]! + console.log(`${name}: median=${median} range=${values[0]}..${values.at(-1)}`) + console.log(`METRIC ${name}=${median}`) +} diff --git a/packages/voice/src/animation.ts b/packages/voice/src/animation.ts new file mode 100644 index 0000000000..777cc77843 --- /dev/null +++ b/packages/voice/src/animation.ts @@ -0,0 +1,49 @@ +export type TextReveal = { readonly offset: number; readonly at: number } + +const CONNECTION_TRANSITION_MS = 480 +const REVEAL_DURATION_MS = 380 +const REVEAL_STAGGER_MS = 32 +const REVEAL_MAX_QUEUE_MS = 160 +export const REVEAL_WORD_LIMIT = 24 + +export function textRevealOpacity(now: number, start: number) { + const progress = Math.max(0, Math.min(1, (now - start) / REVEAL_DURATION_MS)) + const eased = 1 - (1 - progress) ** 2 + return 0.16 + eased * 0.84 +} + +export function transcriptionPulse(now: number) { + return 0.5 + Math.sin(now / 220) * 0.5 +} + +export function connectionMeterLevels(now: number, level: number, connectedAt?: number, startedAt = 0) { + const transition = connectedAt === undefined ? 0 : Math.max(0, Math.min(1, (now - connectedAt) / CONNECTION_TRANSITION_MS)) + const head = ((now - startedAt) / 180) % 4 + return [0, 1, 2, 3].map((index) => { + const distance = Math.abs(index - head) + const wrappedDistance = Math.min(distance, 4 - distance) + const loading = 0.08 + Math.max(0, Math.cos((wrappedDistance / 1.5) * (Math.PI / 2))) * 0.92 + const connected = level * (0.72 + Math.sin(now / 240 + index * 1.4) * 0.28) + return loading + (connected - loading) * transition + }) +} + +export function connectionTransitioning(now: number, connectedAt?: number) { + return connectedAt === undefined || now - connectedAt < CONNECTION_TRANSITION_MS +} + +export function scheduleTextReveal(previous: string, delta: string, now: number, revealAt: number) { + const text = previous + delta + const offsets = Array.from(delta.matchAll(/\S+/g)) + .map((match) => previous.length + match.index) + .filter((offset) => offset === 0 || /\s/.test(text[offset - 1] ?? "")) + .slice(-REVEAL_WORD_LIMIT) + const start = Math.min(Math.max(revealAt, now), now + REVEAL_MAX_QUEUE_MS) + const reveals = offsets.map((offset, word) => ({ offset, at: start + word * REVEAL_STAGGER_MS })) + const nextRevealAt = reveals.length === 0 ? revealAt : reveals.at(-1)!.at + REVEAL_STAGGER_MS + return { + reveals, + nextRevealAt, + animationEndsAt: reveals.length === 0 ? now : reveals.at(-1)!.at + REVEAL_DURATION_MS, + } +} diff --git a/packages/voice/src/audio-device.ts b/packages/voice/src/audio-device.ts new file mode 100644 index 0000000000..e9d617437c --- /dev/null +++ b/packages/voice/src/audio-device.ts @@ -0,0 +1,136 @@ +import { mkdir } from "node:fs/promises" +import { PCM_SAMPLE_RATE } from "./pcm" + +export type AudioDevice = { + readonly fullDuplex: boolean + readonly mode: string + start(onInput: (audio: Buffer) => void, onMeta: (text: string) => void): Promise + write(audio: Buffer): void + flush(): void + close(): void +} + +export async function createAudioDevice(options: { + readonly duplex: boolean + readonly speakers: boolean + readonly debug: boolean +}): Promise { + const helper = await prepareHelper() + if (helper) return appleDevice(helper, options) + return soxDevice(options) +} + +function appleDevice( + binary: string, + options: { readonly duplex: boolean; readonly speakers: boolean }, +): AudioDevice { + let process: ReturnType | undefined + let sink: import("bun").FileSink | undefined + return { + fullDuplex: options.speakers || options.duplex, + mode: options.speakers ? "duplex+aec" : options.duplex ? "duplex" : "half-duplex", + async start(onInput, onMeta) { + if (process) return + const child = Bun.spawn([binary, ...(options.speakers ? ["--aec"] : [])], { + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }) + process = child + sink = child.stdin + void forwardLines(child.stderr, onMeta) + for await (const chunk of child.stdout) onInput(Buffer.from(chunk)) + process = undefined + sink = undefined + }, + write(audio) { + void sink?.write(audio) + void sink?.flush() + }, + flush() { + process?.kill("SIGUSR1") + }, + close() { + process?.kill() + process = undefined + sink = undefined + }, + } +} + +function soxDevice(options: { readonly duplex: boolean; readonly debug: boolean }): AudioDevice { + const format = [ + "-q", + "-t", + "raw", + "-r", + String(PCM_SAMPLE_RATE), + "-e", + "signed-integer", + "-b", + "16", + "-c", + "1", + ] + let recorder: ReturnType | undefined + let player: ReturnType | undefined + let sink: import("bun").FileSink | undefined + let onMeta: ((text: string) => void) | undefined + return { + fullDuplex: options.duplex, + mode: options.duplex ? "duplex (sox)" : "half-duplex (sox)", + async start(onInput, report) { + if (recorder) return + onMeta = report + const child = Bun.spawn(["rec", ...format, "-"], { stdout: "pipe", stderr: "ignore" }) + recorder = child + for await (const chunk of child.stdout) onInput(Buffer.from(chunk)) + recorder = undefined + }, + write(audio) { + if (!player) { + const child = Bun.spawn(["play", ...format, "-"], { stdin: "pipe", stderr: "ignore" }) + player = child + sink = child.stdin + if (options.debug) void child.exited.then((code) => onMeta?.(`[debug] play exited (${code})`)) + } + void sink?.write(audio) + void sink?.flush() + }, + flush() { + player?.kill() + player = undefined + sink = undefined + }, + close() { + recorder?.kill() + player?.kill() + recorder = undefined + player = undefined + sink = undefined + }, + } +} + +async function forwardLines(stream: ReadableStream, onLine: (line: string) => void) { + const decoder = new TextDecoder() + let remainder = "" + for await (const chunk of stream) { + const lines = (remainder + decoder.decode(chunk, { stream: true })).split("\n") + remainder = lines.pop() ?? "" + lines.filter((line) => line.trim()).forEach((line) => onLine(line.trim())) + } + remainder += decoder.decode() + if (remainder.trim()) onLine(remainder.trim()) +} + +async function prepareHelper() { + if (process.platform !== "darwin") return undefined + const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url)) + const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url)) + if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary + await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true }) + const compile = Bun.spawn(["swiftc", "-O", source, "-o", binary], { stdout: "ignore", stderr: "pipe" }) + if ((await compile.exited) === 0) return binary + return undefined +} diff --git a/packages/voice/src/audio-jitter-buffer.ts b/packages/voice/src/audio-jitter-buffer.ts new file mode 100644 index 0000000000..ca60eb53a0 --- /dev/null +++ b/packages/voice/src/audio-jitter-buffer.ts @@ -0,0 +1,41 @@ +import { PCM_BYTES_PER_MS } from "./pcm" + +export type PlaybackChunk = { + readonly bytes: Buffer + readonly gapMs: number +} + +export class AudioJitterBuffer { + private readonly pending: PlaybackChunk[] = [] + private durationMs = 0 + private started = false + + constructor(private readonly targetMs = 500) {} + + push(chunk: PlaybackChunk): ReadonlyArray { + if (this.started) return [chunk] + this.pending.push(chunk) + this.durationMs += chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS + if (this.durationMs < this.targetMs) return [] + this.started = true + return this.drain() + } + + finish(): ReadonlyArray { + if (this.pending.length === 0) return [] + this.started = true + return this.drain() + } + + reset() { + this.pending.length = 0 + this.durationMs = 0 + this.started = false + } + + private drain() { + const chunks = this.pending.splice(0) + this.durationMs = 0 + return chunks + } +} diff --git a/packages/voice/src/audio-session.ts b/packages/voice/src/audio-session.ts new file mode 100644 index 0000000000..cdbfdd921d --- /dev/null +++ b/packages/voice/src/audio-session.ts @@ -0,0 +1,217 @@ +import { createAudioDevice, type AudioDevice } from "./audio-device" +import { AudioJitterBuffer, type PlaybackChunk } from "./audio-jitter-buffer" +import { pcmLevel, PCM_BYTES_PER_MS, PCM_METER_FRAME_MS } from "./pcm" +import type { VoiceAudioTimeline } from "./protocol" + +export type AudioEvent = + | { readonly type: "input"; readonly audio: Buffer; readonly level: number } + | { readonly type: "user.started" } + | { readonly type: "user.stopped" } + | { readonly type: "user.reset" } + | { readonly type: "assistant.level"; readonly level: number; readonly durationMs: number } + | { readonly type: "status"; readonly audio: string } + | { readonly type: "meta"; readonly text: string } + +export type PlaybackOutcome = "played" | "interrupted" | "inaudible" + +export type AudioSession = { + readonly fullDuplex: boolean + readonly microphoneMuted: boolean + readonly speakerMuted: boolean + start(emit: (event: AudioEvent) => void): Promise + toggleMicrophone(): boolean + toggleSpeaker(): boolean + isPlaying(): boolean + play(bytes: Buffer, timeline?: VoiceAudioTimeline): void + finishPlayback(): Promise + flushPlayback(): void + noteUserCommitted(): void + noteUserTranscript(final: boolean): void + close(): void +} + +export async function createAudioSession(options: { + readonly duplex: boolean + readonly speakers: boolean + readonly inputActivity: "local" | "server" + readonly debug: boolean + readonly device?: AudioDevice + readonly trace?: (event: string, data?: Record) => void +}): Promise { + const device = options.device ?? (await createAudioDevice(options)) + const fullDuplex = device.fullDuplex + const playbackBuffer = new AudioJitterBuffer() + let emit: ((event: AudioEvent) => void) | undefined + let microphoneMuted = false + let speakerMuted = false + let playbackEndsAt = 0 + let playbackAudible = false + let playbackDoneTimer: ReturnType | undefined + let playbackCompletion: PromiseWithResolvers | undefined + let outputTimelineEnd: number | undefined + let userFinalizedAt = 0 + let userSpeechTimer: ReturnType | undefined + let userDraftTimer: ReturnType | undefined + let microphoneStarted = false + let closed = false + + const isPlaying = () => Date.now() < playbackEndsAt + + const settlePlayback = (outcome: PlaybackOutcome) => { + if (playbackDoneTimer) clearTimeout(playbackDoneTimer) + playbackDoneTimer = undefined + const completion = playbackCompletion + playbackCompletion = undefined + playbackEndsAt = 0 + playbackAudible = false + playbackBuffer.reset() + outputTimelineEnd = undefined + completion?.resolve(outcome) + } + + const flushPlayback = () => { + device.flush() + settlePlayback("interrupted") + } + + const observeUserAudio = (level: number) => { + if (options.inputActivity !== "local" || level < 0.2 || Date.now() - userFinalizedAt < 500) return + if (userDraftTimer) clearTimeout(userDraftTimer) + userDraftTimer = undefined + emit?.({ type: "user.started" }) + outputTimelineEnd = undefined + if (userSpeechTimer) clearTimeout(userSpeechTimer) + userSpeechTimer = setTimeout(() => { + userSpeechTimer = undefined + emit?.({ type: "user.stopped" }) + userDraftTimer = setTimeout(() => { + userDraftTimer = undefined + emit?.({ type: "user.reset" }) + }, 1_200) + }, 500) + } + + const publishInput = (bytes: Buffer) => { + const level = pcmLevel(bytes) + emit?.({ type: "input", audio: bytes, level }) + observeUserAudio(level) + } + + const start = async (listener: (event: AudioEvent) => void) => { + if (microphoneStarted || closed) return + emit = listener + microphoneStarted = true + emit({ type: "status", audio: device.mode }) + emit({ type: "meta", text: "mic live - start talking" }) + if (!fullDuplex) + emit({ type: "meta", text: "mic mutes while the assistant speaks; press Esc to interrupt" }) + await device.start( + (chunk) => { + if (closed || microphoneMuted || (!fullDuplex && Date.now() < playbackEndsAt + 300)) return + publishInput(chunk) + }, + (text) => { + emit?.({ type: "meta", text }) + options.trace?.("audio.helper", { message: text }) + }, + ) + } + + const writeAudio = (chunk: PlaybackChunk) => { + playbackAudible = true + if (chunk.gapMs > 0) { + emit?.({ type: "assistant.level", level: 0, durationMs: chunk.gapMs }) + device.write(Buffer.alloc(Math.round(chunk.gapMs * PCM_BYTES_PER_MS))) + } + const meterBytes = PCM_METER_FRAME_MS * PCM_BYTES_PER_MS + for (let offset = 0; offset < chunk.bytes.length; offset += meterBytes) { + const window = chunk.bytes.subarray(offset, offset + meterBytes) + emit?.({ type: "assistant.level", level: pcmLevel(window), durationMs: window.length / PCM_BYTES_PER_MS }) + } + device.write(chunk.bytes) + playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + chunk.gapMs + chunk.bytes.length / PCM_BYTES_PER_MS + } + + const schedulePlaybackCompletion = () => { + if (!playbackCompletion) return + if (!playbackAudible) return settlePlayback("inaudible") + if (playbackDoneTimer) clearTimeout(playbackDoneTimer) + playbackDoneTimer = setTimeout( + () => settlePlayback("played"), + Math.max(0, playbackEndsAt - Date.now()) + 180, + ) + } + + return { + fullDuplex, + get microphoneMuted() { + return microphoneMuted + }, + get speakerMuted() { + return speakerMuted + }, + start, + toggleMicrophone() { + microphoneMuted = !microphoneMuted + if (userSpeechTimer) clearTimeout(userSpeechTimer) + userSpeechTimer = undefined + emit?.({ type: "user.stopped" }) + if (microphoneMuted) emit?.({ type: "user.reset" }) + return microphoneMuted + }, + toggleSpeaker() { + speakerMuted = !speakerMuted + if (speakerMuted) flushPlayback() + return speakerMuted + }, + isPlaying, + play(bytes, timeline) { + if (speakerMuted || closed) return + const timelineGap = timeline && outputTimelineEnd !== undefined ? Math.max(0, timeline.startMs - outputTimelineEnd) : 0 + const gapMs = timelineGap < 2_000 ? timelineGap : 0 + if (options.debug) + options.trace?.("audio.output", { + bytes: bytes.length, + durationMs: bytes.length / PCM_BYTES_PER_MS, + level: pcmLevel(bytes), + timelineStart: timeline?.startMs, + timelineEnd: timeline?.endMs, + timelineGap, + gapMs, + }) + outputTimelineEnd = timeline?.endMs + playbackBuffer.push({ bytes, gapMs }).forEach(writeAudio) + schedulePlaybackCompletion() + }, + finishPlayback() { + if (playbackCompletion) return playbackCompletion.promise + playbackCompletion = Promise.withResolvers() + playbackBuffer.finish().forEach(writeAudio) + schedulePlaybackCompletion() + return playbackCompletion?.promise ?? Promise.resolve("inaudible") + }, + flushPlayback, + noteUserCommitted() { + if (userDraftTimer) clearTimeout(userDraftTimer) + userDraftTimer = undefined + }, + noteUserTranscript(final) { + if (userDraftTimer) clearTimeout(userDraftTimer) + userDraftTimer = undefined + if (!final) return + if (userSpeechTimer) clearTimeout(userSpeechTimer) + userSpeechTimer = undefined + userFinalizedAt = Date.now() + emit?.({ type: "user.stopped" }) + }, + close() { + if (closed) return + closed = true + device.close() + if (userSpeechTimer) clearTimeout(userSpeechTimer) + if (userDraftTimer) clearTimeout(userDraftTimer) + settlePlayback("interrupted") + emit = undefined + }, + } +} diff --git a/packages/voice/src/completion-store.ts b/packages/voice/src/completion-store.ts new file mode 100644 index 0000000000..6cf6c418f7 --- /dev/null +++ b/packages/voice/src/completion-store.ts @@ -0,0 +1,109 @@ +import { mkdir, readdir, rename, rm } from "node:fs/promises" +import { homedir } from "node:os" +import { dirname, join } from "node:path" +import { Option, Schema } from "effect" +import { OpenCodeNotification } from "./opencode-notification" +import { promptKey, type PromptHandle } from "./prompt-handle" + +export type CompletionHandle = PromptHandle +export type StoredCompletion = + | { readonly status: "admitting"; readonly handle: CompletionHandle; readonly text: string } + | { readonly status: "pending"; readonly handle: CompletionHandle } + | { + readonly status: "completed" + readonly handle: CompletionHandle + readonly notification: OpenCodeNotification + } + +const CompletionHandle = Schema.Struct({ sessionID: Schema.String, promptID: Schema.String }) +const StoredCompletion = Schema.Union([ + Schema.Struct({ status: Schema.Literal("admitting"), handle: CompletionHandle, text: Schema.String }), + Schema.Struct({ status: Schema.Literal("pending"), handle: CompletionHandle }), + Schema.Struct({ status: Schema.Literal("completed"), handle: CompletionHandle, notification: OpenCodeNotification }), +]) +const StoredCompletions = Schema.Array(StoredCompletion) + +export async function createCompletionStore( + path = join(process.env["XDG_STATE_HOME"] ?? join(homedir(), ".local", "state"), "opencode", "voice-prompts.json"), +) { + await mkdir(dirname(path), { recursive: true }) + const directory = `${path}.d` + await mkdir(directory, { recursive: true }) + const files = (await readdir(directory)).filter((name) => name.endsWith(".json")) + const decoded = files.length > 0 + ? await Promise.all( + files.map(async (name) => { + const entry = Option.getOrUndefined( + Schema.decodeUnknownOption(StoredCompletion)(await Bun.file(join(directory, name)).json()), + ) + if (!entry) throw new Error(`Invalid voice completion entry: ${join(directory, name)}`) + return entry + }), + ) + : (await Bun.file(path).exists()) + ? Option.getOrUndefined(Schema.decodeUnknownOption(StoredCompletions)(await Bun.file(path).json())) + : [] + if (!decoded) throw new Error(`Invalid voice completion store: ${path}`) + const entries = new Map(decoded.map((entry) => [promptKey(entry.handle), entry])) + let writes = Promise.resolve() + + const enqueue = (operation: () => Promise) => { + const result = writes.then(operation) + writes = result.catch(() => {}) + return result + } + + const save = (entry: StoredCompletion) => { + return enqueue(async () => { + const target = completionPath(directory, entry.handle) + const temporary = `${target}.${process.pid}.${crypto.randomUUID()}.tmp` + await Bun.write(temporary, JSON.stringify(entry)) + await rename(temporary, target) + }) + } + + if (files.length === 0 && decoded.length > 0) { + await Promise.all(decoded.map(save)) + await rm(path, { force: true }) + } + if (files.length > 0) { + const existing = new Set(files) + await Promise.all(decoded.filter((entry) => !existing.has(completionFilename(entry.handle))).map(save)) + const current = new Set(decoded.map((entry) => completionFilename(entry.handle))) + await Promise.all(files.filter((name) => !current.has(name)).map((name) => rm(join(directory, name), { force: true }))) + } + + return { + entries: () => [...entries.entries()].toSorted(([left], [right]) => left.localeCompare(right)).map(([, entry]) => entry), + admitting(handle: CompletionHandle, text: string) { + const entry = { status: "admitting", handle, text } as const + entries.set(promptKey(handle), entry) + return save(entry) + }, + pending(handle: CompletionHandle) { + const entry = { status: "pending", handle } as const + entries.set(promptKey(handle), entry) + return save(entry) + }, + completed(handle: CompletionHandle, notification: OpenCodeNotification) { + const entry = { status: "completed", handle, notification } as const + entries.set(promptKey(handle), entry) + return save(entry) + }, + remove(handle: CompletionHandle) { + if (!entries.delete(promptKey(handle))) return writes + return enqueue(() => rm(completionPath(directory, handle), { force: true })) + }, + close: () => writes, + } +} + +export type CompletionStore = Awaited> + +function completionFilename(handle: CompletionHandle) { + return `${encodeURIComponent(promptKey(handle))}.json` +} + +function completionPath(directory: string, handle: CompletionHandle) { + return join(directory, completionFilename(handle)) +} diff --git a/packages/voice/src/duplex-audio.swift b/packages/voice/src/duplex-audio.swift index d6f945998b..682c8bd81f 100644 --- a/packages/voice/src/duplex-audio.swift +++ b/packages/voice/src/duplex-audio.swift @@ -20,35 +20,123 @@ let stderr = FileHandle.standardError func log(_ message: String) { stderr.write(Data("[audio] \(message)\n".utf8)) } final class SpeakerQueue { - private var data = Data() + private static let capacity = 24_000 * 2 * 30 + private var data = [UInt8](repeating: 0, count: capacity) + private var readIndex = 0 + private var writeIndex = 0 + private var count = 0 private let lock = NSLock() func push(_ chunk: Data) { - lock.lock() - data.append(chunk) - lock.unlock() - } - func pop(frames: Int) -> [Int16] { + if chunk.isEmpty { return } lock.lock() defer { lock.unlock() } - let bytes = min(frames * 2, data.count - data.count % 2) - var samples = [Int16](repeating: 0, count: frames) - data.prefix(bytes).withUnsafeBytes { raw in - let int16 = raw.bindMemory(to: Int16.self) - for i in 0..) { + lock.lock() + defer { lock.unlock() } + let samples = min(frames, count / 2) + for index in 0.. OSStatus in - let samples = queue.pop(frames: Int(frameCount)) let out = UnsafeMutableAudioBufferListPointer(audioBufferList)[0].mData!.assumingMemoryBound(to: Float.self) - for i in 0.. 0, micFormat.channelCount > 0, let monoFormat = AVAudioFormat( + commonFormat: .pcmFormatFloat32, sampleRate: micFormat.sampleRate, channels: 1, interleaved: false) + else { + log("microphone route is not ready — retrying") + scheduleRebuild("audio engines unavailable — retrying") + return + } guard let converter = AVAudioConverter(from: monoFormat, to: captureFormat) else { - log("cannot convert \(micFormat.sampleRate)Hz to 24kHz") - exit(2) + log("cannot convert \(micFormat.sampleRate)Hz to 24kHz — retrying") + scheduleRebuild("audio engines unavailable — retrying") + return } engine.inputNode.installTap(onBus: 0, bufferSize: 2400, format: micFormat) { buffer, _ in @@ -132,13 +230,16 @@ func startInput(voiceProcessing: Bool) { do { try engine.start() + log("input active (\(Int(micFormat.sampleRate))Hz, \(micFormat.channelCount)ch, echo cancellation \(voiceProcessing ? "on" : "off"))") } catch { if voiceProcessing { log("input engine failed with voice processing (\(error)) — retrying without") + useAEC = false return startInput(voiceProcessing: false) } log("input engine failed: \(error)") - exit(2) + scheduleRebuild("audio engines unavailable — retrying") + return } // Watchdog: on some devices the voice-processed tap simply never fires. @@ -147,6 +248,7 @@ func startInput(voiceProcessing: Bool) { DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { if tapCount > 0 { return } log("voice-processed mic delivered nothing — restarting without echo cancellation") + useAEC = false startInput(voiceProcessing: false) } } @@ -155,28 +257,19 @@ func startInput(voiceProcessing: Bool) { // Voice processing is opt-in (--aec): it is only needed on speakers, and on // some machines (observed with Bluetooth headsets active) the VP engine binds // to the wrong capture device entirely, delivering noise instead of the mic. -let wantAEC = CommandLine.arguments.contains("--aec") - // Mic first: activating the mic flips Bluetooth headsets from music mode to // headset mode, reconfiguring the output device. Starting output afterwards // (and rebuilding on any route change below) keeps playback on the live device. -startInput(voiceProcessing: wantAEC) +startInput(voiceProcessing: useAEC) startOutput() // Device switches (Bluetooth profile flips, headphones plugged/unplugged, // default device changes) stop engines silently. Rebuild both, debounced. -var rebuildScheduled = false NotificationCenter.default.addObserver( forName: .AVAudioEngineConfigurationChange, object: nil, queue: .main ) { _ in - if rebuildScheduled { return } - rebuildScheduled = true - DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { - rebuildScheduled = false - log("audio route changed — rebuilding engines") - startInput(voiceProcessing: wantAEC) - startOutput() - } + if Date() < ignoreRouteChangesUntil { return } + scheduleRebuild("audio route changed — rebuilding engines") } signal(SIGUSR1, SIG_IGN) diff --git a/packages/voice/src/notification-router.ts b/packages/voice/src/notification-router.ts new file mode 100644 index 0000000000..7ed2032970 --- /dev/null +++ b/packages/voice/src/notification-router.ts @@ -0,0 +1,48 @@ +import type { CompletionReceipt, OpenCodeAnnouncement } from "./opencode-notification" +import { promptKey } from "./prompt-handle" + +export type RoutedAnnouncement = { + readonly announcement: OpenCodeAnnouncement + readonly promptID?: string + readonly delegationID?: string +} + +export function createNotificationRouter(deliver: (announcement: RoutedAnnouncement) => void) { + const delegations = new Map() + const deferred: OpenCodeAnnouncement[] = [] + let pendingDelegatedTools = 0 + + const route = (announcement: OpenCodeAnnouncement) => { + const prompt = "prompt_id" in announcement.notification + ? { sessionID: announcement.notification.session_id, promptID: announcement.notification.prompt_id } + : undefined + const promptID = prompt?.promptID + if (prompt && pendingDelegatedTools > 0 && !delegations.has(promptKey(prompt))) { + deferred.push(announcement) + return + } + deliver({ + announcement, + promptID, + delegationID: prompt ? delegations.get(promptKey(prompt)) : undefined, + }) + } + + return { + route, + beginDelegatedTool() { + pendingDelegatedTools += 1 + }, + finishDelegatedTool( + delegationID: string, + admittedPrompt?: { readonly sessionID: string; readonly promptID: string }, + ) { + if (admittedPrompt) delegations.set(promptKey(admittedPrompt), delegationID) + pendingDelegatedTools -= 1 + if (pendingDelegatedTools === 0) deferred.splice(0).forEach(route) + }, + acknowledged(receipt: CompletionReceipt) { + delegations.delete(promptKey(receipt)) + }, + } +} diff --git a/packages/voice/src/opencode-notification.ts b/packages/voice/src/opencode-notification.ts new file mode 100644 index 0000000000..f6a371c076 --- /dev/null +++ b/packages/voice/src/opencode-notification.ts @@ -0,0 +1,97 @@ +import { Schema } from "effect" +import type { PromptHandle } from "./prompt-handle" + +const Completed = Schema.Struct({ + type: Schema.Literal("opencode.prompt.completed"), + session_id: Schema.String, + prompt_id: Schema.String, + status: Schema.Literals(["completed", "failed"]), + text: Schema.String, + error: Schema.optional(Schema.String), +}) + +const Failed = Schema.Struct({ + type: Schema.Literal("opencode.prompt.failed"), + session_id: Schema.String, + prompt_id: Schema.String, + status: Schema.Literal("failed"), + error: Schema.String, +}) + +const PermissionBlocked = Schema.Struct({ + type: Schema.Literal("opencode.prompt.blocked"), + prompt_id: Schema.String, + blocker: Schema.Literal("permission"), + session_id: Schema.String, + request_id: Schema.String, + action: Schema.String, + resources: Schema.Unknown, +}) + +const QuestionBlocked = Schema.Struct({ + type: Schema.Literal("opencode.prompt.blocked"), + prompt_id: Schema.String, + blocker: Schema.Literal("question"), + session_id: Schema.String, + request_id: Schema.String, + questions: Schema.Unknown, +}) + +const FormBlocked = Schema.Struct({ + type: Schema.Literal("opencode.prompt.blocked"), + prompt_id: Schema.String, + blocker: Schema.Literal("form"), + session_id: Schema.String, + form_id: Schema.String, + title: Schema.String, + fields: Schema.Unknown, +}) + +const EventsFailed = Schema.Struct({ + type: Schema.Literal("opencode.events.failed"), + error: Schema.String, +}) + +export const OpenCodeNotification = Schema.Union([ + Completed, + Failed, + PermissionBlocked, + QuestionBlocked, + FormBlocked, + EventsFailed, +]) +export type OpenCodeNotification = typeof OpenCodeNotification.Type +export type OpenCodePromptBlocked = Extract + +export type CompletionReceipt = PromptHandle + +export type OpenCodeAnnouncement = { + readonly notification: OpenCodeNotification + readonly receipt?: CompletionReceipt +} + +export function openCodeAnnouncementText(notification: OpenCodeNotification) { + const text = (() => { + switch (notification.type) { + case "opencode.prompt.completed": + return `OpenCode prompt ${notification.status}: ${notification.text}${notification.error ? ` Error: ${notification.error}` : ""}` + case "opencode.prompt.failed": + return `OpenCode prompt failed: ${notification.error}` + case "opencode.prompt.blocked": + if (notification.blocker === "permission") + return `OpenCode needs permission for ${notification.action}: ${compact(notification.resources)}` + if (notification.blocker === "question") + return `OpenCode needs the user to answer: ${compact(notification.questions)}` + return `OpenCode needs the user to complete the form “${notification.title}”: ${compact(notification.fields)}` + case "opencode.events.failed": + return `OpenCode event delivery failed: ${notification.error}` + } + return "OpenCode has an update." + })() + return text.length > 800 ? text.slice(0, 797) + "..." : text +} + +function compact(value: unknown) { + if (typeof value === "string") return value + return JSON.stringify(value) ?? String(value) +} diff --git a/packages/voice/src/opencode.ts b/packages/voice/src/opencode.ts new file mode 100644 index 0000000000..9b1e6870ff --- /dev/null +++ b/packages/voice/src/opencode.ts @@ -0,0 +1,811 @@ +import { realpathSync } from "node:fs" +import { tmpdir } from "node:os" +import { basename, isAbsolute, relative } from "node:path" +import { ClientError, isSessionNotFoundError, OpenCode } from "@opencode-ai/client/promise" +import type { SessionMessageAssistant, SessionMessageUser, V2Event } from "@opencode-ai/client/promise" +import { Form, Question, SessionMessage } from "@opencode-ai/client/effect" +import { Context, Effect, FiberMap, Layer, ManagedRuntime, Option, Schema, Stream } from "effect" +import { createCompletionStore, type CompletionStore } from "./completion-store" +import type { + CompletionReceipt, + OpenCodeAnnouncement, + OpenCodeNotification, + OpenCodePromptBlocked, +} from "./opencode-notification" +import type { VoiceTool, VoiceToolExecution } from "./protocol" +import type { PromptHandle } from "./prompt-handle" + +type Client = ReturnType +const AdmittedPrompt = Symbol("AdmittedPrompt") +type Tool = { + readonly description: string + readonly parameters: unknown + readonly execute: (input: Record) => Effect.Effect +} +type BridgeApi = { + readonly definitions: ReadonlyArray + readonly execute: (name: string, input: Record) => Effect.Effect + readonly acknowledge: (receipt: CompletionReceipt) => Effect.Effect + readonly close: Effect.Effect +} + +class Bridge extends Context.Service()("@opencode-ai/voice/OpenCodeBridge") {} + +export async function createOpenCodeBridge(options: { + client: Client + directory: string + model: { readonly providerID: string; readonly id: string; readonly variant?: string } + notify: (announcement: OpenCodeAnnouncement) => void + trace?: (event: string, data?: Record) => void + completionStore?: CompletionStore +}) { + const completionStore = options.completionStore ?? (await createCompletionStore()) + const runtime = ManagedRuntime.make(Layer.effect(Bridge, makeBridge({ ...options, completionStore }))) + const bridge = await runtime.runPromise(Bridge) + return { + definitions: bridge.definitions, + execute: (name: string, input: Record) => runtime.runPromise(bridge.execute(name, input)), + acknowledge: (receipt: CompletionReceipt) => runtime.runPromise(bridge.acknowledge(receipt)), + close: async () => { + await runtime.runPromise(bridge.close) + await Promise.race([runtime.dispose(), Bun.sleep(1_000)]) + await completionStore.close() + }, + } +} + +const makeBridge = Effect.fnUntraced(function* (options: { + client: Client + directory: string + model: { readonly providerID: string; readonly id: string; readonly variant?: string } + notify: (announcement: OpenCodeAnnouncement) => void + trace?: (event: string, data?: Record) => void + completionStore: CompletionStore +}) { + const knownProjects = new Set() + const knownSessions = new Set() + const registrations = new Map() + const promoted = new Map() + const latest = new Map() + const announcedBlockers = new Set() + const completions = yield* FiberMap.make() + const eventAbort = yield* Effect.acquireRelease( + Effect.sync(() => new AbortController()), + (controller) => Effect.sync(() => controller.abort()), + ) + + const notify = (notification: OpenCodeNotification, receipt?: CompletionReceipt) => + Effect.sync(() => options.notify({ notification, receipt })) + const trace = (event: string, data: Record) => Effect.sync(() => options.trace?.(event, data)) + const request = (run: (signal: AbortSignal) => PromiseLike) => + Effect.tryPromise({ try: run, catch: (cause) => cause }) + const announceBlocker = (key: string, notification: OpenCodeNotification) => { + if (announcedBlockers.has(key)) return Effect.void + announcedBlockers.add(key) + return notify(notification) + } + const clearRegistration = (handle: PromptHandle) => + Effect.sync(() => { + registrations.delete(handle.promptID) + if (promoted.get(handle.sessionID) === handle.promptID) promoted.delete(handle.sessionID) + if (latest.get(handle.sessionID) === handle.promptID) latest.delete(handle.sessionID) + }) + + const listProjects = Effect.fnUntraced(function* () { + const seen = new Set() + const temporary = realpathSync(tmpdir()) + return (yield* request((signal) => options.client.project.list({ signal }))) + .sort((a, b) => b.time.updated - a.time.updated) + .filter((project) => { + const fromTemporary = relative(temporary, project.worktree) + if ( + project.id === "global" || + fromTemporary === "" || + (!fromTemporary.startsWith("..") && !isAbsolute(fromTemporary)) || + seen.has(project.worktree) + ) + return false + seen.add(project.worktree) + return true + }) + }) + + const projectDirectory = Effect.fnUntraced(function* (projectID: string) { + if (!knownProjects.has(projectID)) return undefined + return (yield* listProjects()).find((project) => project.id === projectID)?.worktree + }) + + const complete = Effect.fnUntraced(function* (handle: PromptHandle) { + yield* trace("opencode.wait.started", handle) + yield* waitForIdle(handle) + const reply = yield* request((signal) => finalReply(options.client, handle, signal)) + yield* trace("opencode.wait.completed", { ...handle, replyID: reply?.id, error: reply?.error?.message }) + const notification: OpenCodeNotification = { + type: "opencode.prompt.completed", + session_id: handle.sessionID, + prompt_id: handle.promptID, + status: reply?.error ? "failed" : "completed", + text: reply + ? assistantText(reply) || "OpenCode finished without a text reply." + : "OpenCode finished without a reply.", + error: reply?.error?.message, + } + yield* request(() => options.completionStore.completed(handle, notification)) + yield* notify(notification, handle) + }) + + const waitForIdle = Effect.fnUntraced(function* (handle: PromptHandle, attempt = 1): Effect.fn.Return { + return yield* request((signal) => options.client.session.wait({ sessionID: handle.sessionID }, { signal })).pipe( + Effect.catch((error) => { + if (!(error instanceof ClientError) || error.reason !== "Transport") return Effect.fail(error) + return trace("opencode.wait.retrying", { + ...handle, + attempt, + reason: error.reason, + cause: error.cause instanceof Error ? error.cause.name : typeof error.cause, + }).pipe(Effect.andThen(Effect.sleep("1 second")), Effect.andThen(waitForIdle(handle, attempt + 1))) + }), + ) + }) + + const register = Effect.fnUntraced(function* (handle: PromptHandle) { + yield* complete(handle).pipe( + Effect.catch((error) => { + const notification: OpenCodeNotification = { + type: "opencode.prompt.failed", + session_id: handle.sessionID, + prompt_id: handle.promptID, + status: "failed", + error: String(error), + } + return request(() => options.completionStore.completed(handle, notification)).pipe( + Effect.andThen(notify(notification, handle)), + ) + }), + Effect.ensuring(clearRegistration(handle)), + FiberMap.run(completions, handle.promptID, { onlyIfMissing: true, startImmediately: true }), + ) + }) + + const restoreBlockers = Effect.fnUntraced(function* (handle: PromptHandle) { + const [permissions, questions, forms] = yield* Effect.all( + [ + request((signal) => options.client.permission.list({ sessionID: handle.sessionID }, { signal })), + request((signal) => options.client.question.list({ sessionID: handle.sessionID }, { signal })), + request((signal) => options.client.form.list({ sessionID: handle.sessionID }, { signal })), + ], + { concurrency: "unbounded" }, + ) + yield* Effect.forEach( + [ + ...permissions.map((item) => ({ + key: `permission:${item.id}`, + notification: { + type: "opencode.prompt.blocked", + prompt_id: handle.promptID, + blocker: "permission", + session_id: handle.sessionID, + request_id: item.id, + action: item.action, + resources: item.resources, + } satisfies OpenCodeNotification, + })), + ...questions.map((item) => ({ + key: `question:${item.id}`, + notification: { + type: "opencode.prompt.blocked", + prompt_id: handle.promptID, + blocker: "question", + session_id: handle.sessionID, + request_id: item.id, + questions: item.questions, + } satisfies OpenCodeNotification, + })), + ...forms.map((item) => ({ + key: `form:${item.id}`, + notification: { + type: "opencode.prompt.blocked", + prompt_id: handle.promptID, + blocker: "form", + session_id: handle.sessionID, + form_id: item.id, + title: item.title, + fields: item.fields, + } satisfies OpenCodeNotification, + })), + ], + (item) => announceBlocker(item.key, item.notification), + { discard: true }, + ) + }) + + const admit = Effect.fnUntraced(function* (sessionID: string, text: string) { + const handle = { sessionID, promptID: SessionMessage.ID.create() } + registrations.set(handle.promptID, handle) + latest.set(sessionID, handle.promptID) + yield* trace("opencode.prompt.admitting", handle) + yield* request(() => options.completionStore.admitting(handle, text)) + const admitted = yield* request((signal) => + options.client.session.prompt({ sessionID, id: handle.promptID, text }, { signal }), + ).pipe( + Effect.tapError(() => + request(() => options.completionStore.remove(handle)).pipe( + Effect.andThen(clearRegistration(handle)), + ), + ), + ) + knownSessions.add(sessionID) + yield* request(() => options.completionStore.pending(handle)) + yield* trace("opencode.prompt.admitted", { ...handle, admittedSeq: admitted.admittedSeq }) + yield* register(handle) + return { + status: "started", + session_id: sessionID, + prompt_id: admitted.id, + notification: "registered", + message: "OpenCode accepted the prompt and will send a completion notification. Continue the conversation now.", + [AdmittedPrompt]: handle, + } + }) + + const start = Effect.fnUntraced(function* (text: string, projectID?: string) { + const directory = projectID ? yield* projectDirectory(projectID) : options.directory + if (!directory) return toolError("Use a project ID returned by find_projects.") + const session = yield* request((signal) => + options.client.session.create({ location: { directory }, model: options.model }, { signal }), + ) + return yield* admit(session.id, text) + }) + + const tools: Record = { + find_projects: { + description: "Find known OpenCode projects by display name. Returns opaque IDs, never filesystem paths.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + query: { type: ["string", "null"], description: "Name fragment, or null for recent projects." }, + limit: { type: "integer", minimum: 1, maximum: 20 }, + }, + required: ["query", "limit"], + }, + execute: Effect.fnUntraced(function* (input) { + const query = typeof input["query"] === "string" ? input["query"].toLowerCase() : undefined + const limit = typeof input["limit"] === "number" ? input["limit"] : 10 + const projects = (yield* listProjects()) + .filter((project) => !query || projectLabel(project).toLowerCase().includes(query)) + .slice(0, limit) + projects.forEach((project) => knownProjects.add(project.id)) + return { + status: "ok", + projects: projects.map((project) => ({ + id: project.id, + name: projectLabel(project), + directories: 1 + project.sandboxes.length, + updated: new Date(project.time.updated).toISOString(), + })), + } + }), + }, + find_sessions: { + description: + "Find root OpenCode sessions by title and recency. Search the launch project by default; use all_projects only when requested or the local search misses.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + query: { type: ["string", "null"], description: "Title words, or null for recent sessions." }, + scope: { type: "string", enum: ["current_project", "all_projects"] }, + recency: { type: "string", enum: ["day", "week", "month", "any"] }, + limit: { type: "integer", minimum: 1, maximum: 20 }, + }, + required: ["query", "scope", "recency", "limit"], + }, + execute: Effect.fnUntraced(function* (input) { + const query = typeof input["query"] === "string" ? input["query"] : undefined + const scope = input["scope"] === "all_projects" ? "all_projects" : "current_project" + const recency = typeof input["recency"] === "string" ? input["recency"] : "any" + const limit = typeof input["limit"] === "number" ? input["limit"] : 10 + const durations: Record = { day: 86_400_000, week: 604_800_000, month: 2_592_000_000 } + const threshold = recency === "any" ? 0 : Date.now() - (durations[recency] ?? 0) + const [result, projectList] = yield* Effect.all( + [ + request((signal) => + options.client.session.list( + { + ...(scope === "current_project" ? { directory: options.directory } : {}), + search: query, + parentID: null, + limit: recency === "any" ? limit : Math.min(limit * 5, 100), + order: "desc", + }, + { signal }, + ), + ), + listProjects(), + ], + { concurrency: "unbounded" }, + ) + const projects = new Map(projectList.map((project) => [project.id, projectLabel(project)])) + const sessions = result.data + .filter((session) => projects.has(session.projectID) && session.time.updated >= threshold) + .slice(0, limit) + sessions.forEach((session) => knownSessions.add(session.id)) + return { + status: "ok", + scope, + sessions: sessions.map((session) => ({ + id: session.id, + title: session.title, + project: projects.get(session.projectID) ?? "project", + updated: new Date(session.time.updated).toISOString(), + })), + } + }), + }, + read_session: { + description: + "Read bounded recent user and assistant text from a session returned by find_sessions or start_session. This never prompts or wakes the coding agent.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + session_id: { type: "string", description: "Session ID returned by a previous voice tool." }, + limit: { type: "integer", minimum: 1, maximum: 20 }, + }, + required: ["session_id", "limit"], + }, + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.") + const limit = typeof input["limit"] === "number" ? input["limit"] : 10 + const [session, messages] = yield* Effect.all( + [ + request((signal) => options.client.session.get({ sessionID }, { signal })), + request((signal) => options.client.message.list({ sessionID, order: "desc", limit }, { signal })), + ], + { concurrency: "unbounded" }, + ) + const latestAssistant = messages.data.find( + (message): message is SessionMessageAssistant => message.type === "assistant", + ) + return { + status: "ok", + title: session.title, + running: latestAssistant ? !latestAssistant.time.completed : false, + messages: messages.data + .toReversed() + .flatMap((message) => + message.type === "user" || message.type === "assistant" ? [sessionMessage(message)] : [], + ), + } + }), + }, + rename_session: { + description: + "Rename a session returned by find_sessions or start_session after the user requests a new title. This changes only the display title and does not prompt, wake, or interrupt the coding agent.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + session_id: { type: "string", description: "Session ID returned by a previous voice tool." }, + title: { type: "string", description: "The complete new session title." }, + }, + required: ["session_id", "title"], + }, + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + const title = requireString(input, "title")?.trim() + if (!sessionID) return toolError("Use a session ID returned by find_sessions or start_session.") + if (!title) return toolError("A non-empty session title is required.") + const session = yield* request((signal) => options.client.session.get({ sessionID }, { signal })) + if (session.title === title) + return { status: "unchanged", session_id: sessionID, previous_title: session.title, title } + yield* request((signal) => options.client.session.rename({ sessionID, title }, { signal })) + return { status: "renamed", session_id: sessionID, previous_title: session.title, title } + }), + }, + start_session: { + description: + "Create an OpenCode session, admit its first prompt, and register a one-shot completion notification. Omit project_id to use the launch project.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + text: { type: "string", description: "Clear instruction for the coding agent." }, + project_id: { type: ["string", "null"], description: "Opaque project ID, or null for the launch project." }, + }, + required: ["text", "project_id"], + }, + execute: (input) => { + const text = requireString(input, "text") + if (!text) return Effect.succeed(toolError("Task text is required.")) + return start(text, typeof input["project_id"] === "string" ? input["project_id"] : undefined) + }, + }, + prompt_session: { + description: + "Admit a prompt into a discovered or previously started OpenCode session and register a one-shot completion notification.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + session_id: { type: "string", description: "Session ID returned by a previous voice tool." }, + text: { type: "string", description: "Clear instruction for the coding agent." }, + }, + required: ["session_id", "text"], + }, + execute: (input) => { + const sessionID = knownSession(input, knownSessions) + const text = requireString(input, "text") + if (!sessionID) return Effect.succeed(toolError("Use a session ID returned by find_sessions or start_session.")) + if (!text) return Effect.succeed(toolError("Task text is required.")) + return admit(sessionID, text) + }, + }, + interrupt_session: { + description: "Interrupt one OpenCode session. Call only after the user explicitly confirms the interruption.", + parameters: sessionParameters(), + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.") + yield* request((signal) => options.client.session.interrupt({ sessionID }, { signal })) + return { status: "interrupted", session_id: sessionID } + }), + }, + list_pending_permissions: { + description: "List permission requests blocking one OpenCode session.", + parameters: sessionParameters(), + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + if (!sessionID) return toolError("Use a session ID returned by a previous voice tool.") + const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal })) + return { + status: "ok", + requests: requests.map((item) => ({ id: item.id, action: item.action, resources: item.resources })), + } + }), + }, + reply_permission: { + description: + "Allow once or reject a pending permission after stating the action and resources and receiving the user's explicit decision.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + session_id: { type: "string" }, + request_id: { type: "string" }, + decision: { type: "string", enum: ["allow_once", "reject"] }, + }, + required: ["session_id", "request_id", "decision"], + }, + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + const requestID = requireString(input, "request_id") + const decision = input["decision"] + if (!sessionID || !requestID || (decision !== "allow_once" && decision !== "reject")) + return toolError("A known session, request ID, and valid decision are required.") + const requests = yield* request((signal) => options.client.permission.list({ sessionID }, { signal })) + if (!requests.some((item) => item.id === requestID)) + return toolError("That permission request is not pending.", true) + yield* request((signal) => + options.client.permission.reply( + { sessionID, requestID, reply: decision === "allow_once" ? "once" : "reject" }, + { signal }, + ), + ) + return { status: decision === "allow_once" ? "allowed_once" : "rejected", request_id: requestID } + }), + }, + reply_question: { + description: "Reply to questions blocking an OpenCode session after collecting the user's answers.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + session_id: { type: "string" }, + request_id: { type: "string" }, + answers: { + type: "array", + description: "One string array per question, preserving question order.", + items: { type: "array", items: { type: "string" } }, + }, + }, + required: ["session_id", "request_id", "answers"], + }, + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + const requestID = requireString(input, "request_id") + const answers = Option.getOrUndefined(decodeQuestionAnswers(input["answers"])) + if (!sessionID || !requestID || !answers) + return toolError("A known session, request ID, and valid answers are required.") + yield* request((signal) => options.client.question.reply({ sessionID, requestID, answers }, { signal })) + return { status: "answered", request_id: requestID } + }), + }, + reject_question: { + description: "Reject a pending OpenCode question after the user explicitly declines to answer.", + parameters: requestParameters(), + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + const requestID = requireString(input, "request_id") + if (!sessionID || !requestID) return toolError("A known session and request ID are required.") + yield* request((signal) => options.client.question.reject({ sessionID, requestID }, { signal })) + return { status: "rejected", request_id: requestID } + }), + }, + reply_form: { + description: "Submit values for a form blocking an OpenCode session after collecting them from the user.", + parameters: { + type: "object", + additionalProperties: false, + properties: { + session_id: { type: "string" }, + form_id: { type: "string" }, + answer: { type: "object", additionalProperties: true }, + }, + required: ["session_id", "form_id", "answer"], + }, + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + const formID = requireString(input, "form_id") + const answer = Option.getOrUndefined(decodeFormAnswer(input["answer"])) + if (!sessionID || !formID || !answer) + return toolError("A known session, form ID, and valid answer are required.") + yield* request((signal) => options.client.form.reply({ sessionID, formID, answer }, { signal })) + return { status: "answered", form_id: formID } + }), + }, + cancel_form: { + description: "Cancel a pending OpenCode form after the user explicitly declines to complete it.", + parameters: { + type: "object", + additionalProperties: false, + properties: { session_id: { type: "string" }, form_id: { type: "string" } }, + required: ["session_id", "form_id"], + }, + execute: Effect.fnUntraced(function* (input) { + const sessionID = knownSession(input, knownSessions) + const formID = requireString(input, "form_id") + if (!sessionID || !formID) return toolError("A known session and form ID are required.") + yield* request((signal) => options.client.form.cancel({ sessionID, formID }, { signal })) + return { status: "cancelled", form_id: formID } + }), + }, + } + + const definitions = Object.entries(tools).map( + ([name, tool]) => + ({ type: "function", name, description: tool.description, parameters: tool.parameters }) satisfies VoiceTool, + ) + + yield* Effect.suspend(() => + Stream.fromAsyncIterable(options.client.event.subscribe({ signal: eventAbort.signal }), (cause) => cause).pipe( + Stream.runForEach((event) => { + if (event.type === "session.input.promoted") { + if (!registrations.has(event.data.inputID)) return Effect.void + promoted.set(event.data.sessionID, event.data.inputID) + return trace("opencode.prompt.promoted", { + sessionID: event.data.sessionID, + promptID: event.data.inputID, + }) + } + const sessionID = blockerSession(event) + const promptID = sessionID ? (promoted.get(sessionID) ?? latest.get(sessionID)) : undefined + const blocker = sessionID && promptID ? blockerNotification(event, sessionID, promptID) : undefined + if (!blocker || !promptID) return Effect.void + options.trace?.("opencode.prompt.blocked", { sessionID, promptID, blocker: blocker["blocker"] }) + if (registrations.has(promptID)) return announceBlocker(blockerKey(blocker), blocker) + return Effect.void + }), + Effect.catch((error) => notify({ type: "opencode.events.failed", error: String(error) })), + Effect.andThen(Effect.sleep("1 second")), + ), + ).pipe(Effect.forever, Effect.forkScoped({ startImmediately: true })) + + yield* Effect.forEach( + options.completionStore.entries(), + Effect.fnUntraced(function* (entry) { + if (entry.status !== "completed") { + const exists = yield* request((signal) => + options.client.session.get({ sessionID: entry.handle.sessionID }, { signal }), + ).pipe( + Effect.as(true), + Effect.catch((error) => { + if (!isSessionNotFoundError(error)) return Effect.fail(error) + return request(() => options.completionStore.remove(entry.handle)).pipe( + Effect.andThen(trace("opencode.prompt.orphaned", entry.handle)), + Effect.as(false), + ) + }), + ) + if (!exists) return + } + knownSessions.add(entry.handle.sessionID) + if (entry.status === "admitting") { + registrations.set(entry.handle.promptID, entry.handle) + latest.set(entry.handle.sessionID, entry.handle.promptID) + yield* trace("opencode.prompt.reconciling", entry.handle) + yield* request((signal) => + options.client.session.prompt( + { sessionID: entry.handle.sessionID, id: entry.handle.promptID, text: entry.text }, + { signal }, + ), + ) + yield* request(() => options.completionStore.pending(entry.handle)) + yield* restoreBlockers(entry.handle) + yield* register(entry.handle) + return + } + if (entry.status === "pending") { + registrations.set(entry.handle.promptID, entry.handle) + latest.set(entry.handle.sessionID, entry.handle.promptID) + yield* trace("opencode.wait.restored", entry.handle) + yield* restoreBlockers(entry.handle) + yield* register(entry.handle) + return + } + yield* trace("opencode.notification.restored", entry.handle) + yield* notify(entry.notification, entry.handle) + }), + { concurrency: 4, discard: true }, + ) + + return Bridge.of({ + definitions, + execute: (name, input) => + (tools[name]?.execute(input) ?? Effect.succeed(toolError(`Unknown tool ${name}.`))).pipe( + Effect.map(toolExecution), + ), + acknowledge: (receipt) => request(() => options.completionStore.remove(receipt)), + close: Effect.all([Effect.sync(() => eventAbort.abort()), FiberMap.clear(completions)], { discard: true }), + }) +}) + +function toolExecution(output: unknown): VoiceToolExecution { + if (!isPromptToolOutput(output)) return { output } + return { + output: Object.fromEntries(Object.entries(output)), + admittedPrompt: output[AdmittedPrompt], + } +} + +function isPromptToolOutput( + output: unknown, +): output is Record & { readonly [AdmittedPrompt]: PromptHandle } { + if (!output || typeof output !== "object" || !(AdmittedPrompt in output)) return false + const handle = output[AdmittedPrompt] + return ( + !!handle && + typeof handle === "object" && + "sessionID" in handle && + typeof handle.sessionID === "string" && + "promptID" in handle && + typeof handle.promptID === "string" + ) +} + +async function finalReply(client: Client, handle: PromptHandle, signal: AbortSignal) { + let reply: SessionMessageAssistant | undefined + let cursor: string | undefined + while (true) { + const page = await client.message.list( + cursor + ? { sessionID: handle.sessionID, limit: 200, cursor } + : { sessionID: handle.sessionID, limit: 200, order: "desc" }, + { signal }, + ) + for (const message of page.data) { + if (message.id === handle.promptID) return reply + if (message.type === "user") reply = undefined + if (message.type === "assistant" && !reply) reply = message + } + cursor = page.cursor.next ?? undefined + if (!cursor) return undefined + } +} + +function assistantText(message: SessionMessageAssistant) { + return message.content + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n") + .slice(0, 4_000) +} + +function sessionMessage(message: SessionMessageUser | SessionMessageAssistant) { + if (message.type === "user") return { role: "user", text: message.text.slice(0, 2_000) } + return { + role: "assistant", + status: message.time.completed ? "completed" : "running", + text: assistantText(message), + tools: message.content + .filter((part) => part.type === "tool") + .map((part) => ({ name: part.name, status: part.state.status })), + } +} + +function blockerSession(event: V2Event) { + if (event.type === "permission.v2.asked" || event.type === "question.v2.asked") return event.data.sessionID + if (event.type === "form.created") return event.data.form.sessionID + return undefined +} + +function blockerNotification(event: V2Event, sessionID: string, promptID: string): OpenCodePromptBlocked | undefined { + if (event.type === "permission.v2.asked") + return { + type: "opencode.prompt.blocked", + prompt_id: promptID, + blocker: "permission", + session_id: sessionID, + request_id: event.data.id, + action: event.data.action, + resources: event.data.resources, + } + if (event.type === "question.v2.asked") + return { + type: "opencode.prompt.blocked", + prompt_id: promptID, + blocker: "question", + session_id: sessionID, + request_id: event.data.id, + questions: event.data.questions, + } + if (event.type === "form.created") + return { + type: "opencode.prompt.blocked", + prompt_id: promptID, + blocker: "form", + session_id: sessionID, + form_id: event.data.form.id, + title: event.data.form.title, + fields: event.data.form.fields, + } + return undefined +} + +function blockerKey(notification: OpenCodePromptBlocked) { + if (notification.blocker === "form") return `form:${notification.form_id}` + return `${notification.blocker}:${notification.request_id}` +} + +function projectLabel(project: { readonly name?: string; readonly worktree: string }) { + return project.name ?? (basename(project.worktree) || "project") +} + +function requireString(input: Record, key: string) { + const value = input[key] + if (typeof value !== "string" || value.trim().length === 0) return undefined + return value +} + +function knownSession(input: Record, known: ReadonlySet) { + const sessionID = requireString(input, "session_id") + if (!sessionID || !known.has(sessionID)) return undefined + return sessionID +} + +const decodeQuestionAnswers = Schema.decodeUnknownOption(Schema.Array(Question.Answer)) +const decodeFormAnswer = Schema.decodeUnknownOption(Form.Answer) + +function sessionParameters() { + return { + type: "object", + additionalProperties: false, + properties: { session_id: { type: "string" } }, + required: ["session_id"], + } +} + +function requestParameters() { + return { + type: "object", + additionalProperties: false, + properties: { session_id: { type: "string" }, request_id: { type: "string" } }, + required: ["session_id", "request_id"], + } +} + +function toolError(message: string, retryable = false) { + return { status: "error", message, retryable } +} diff --git a/packages/voice/src/pcm.ts b/packages/voice/src/pcm.ts new file mode 100644 index 0000000000..99672c0fc9 --- /dev/null +++ b/packages/voice/src/pcm.ts @@ -0,0 +1,19 @@ +export const PCM_SAMPLE_RATE = 24_000 +export const PCM_BYTES_PER_MS = (PCM_SAMPLE_RATE * 2) / 1_000 +export const PCM_METER_FRAME_MS = 1_000 / 60 + +export function pcmLevel(bytes: Buffer) { + const samples = Math.floor(bytes.length / 2) + if (samples === 0) return 0 + const stride = Math.max(1, Math.floor(samples / 1_200)) + let energy = 0 + let count = 0 + for (let index = 0; index < samples; index += stride) { + const sample = bytes.readInt16LE(index * 2) / 32_768 + energy += sample * sample + count += 1 + } + const rms = Math.sqrt(energy / count) + if (rms < 0.008) return 0 + return Math.min(1, Math.sqrt((rms - 0.008) / 0.18)) +} diff --git a/packages/voice/src/plugin.tsx b/packages/voice/src/plugin.tsx new file mode 100644 index 0000000000..5c0a8b1c8a --- /dev/null +++ b/packages/voice/src/plugin.tsx @@ -0,0 +1,37 @@ +import { Plugin } from "@opencode-ai/plugin/v2/tui" +import { createSignal } from "solid-js" + +function VoiceStatus(props: { context: Plugin.Context }) { + const [enabled, setEnabled] = createSignal(false) + + props.context.keymap.layer(() => ({ + mode: "global", + commands: [ + { + id: "voice.smoke.toggle", + title: "Toggle voice plugin", + description: "Toggle the V2 voice plugin smoke-test status", + group: "Voice", + bind: "alt+v", + palette: true, + run() { + setEnabled((value) => !value) + }, + }, + ], + })) + + return ( + + voice plugin: {enabled() ? "on" : "off"} (alt+v) + + ) +} + +export default Plugin.define({ + id: "opencode.voice-smoke", + setup(context) { + context.ui.slot("home.bottom", () => ) + context.ui.slot("sidebar.content", () => ) + }, +}) diff --git a/packages/voice/src/prompt-handle.ts b/packages/voice/src/prompt-handle.ts new file mode 100644 index 0000000000..e9b2ce44f2 --- /dev/null +++ b/packages/voice/src/prompt-handle.ts @@ -0,0 +1,8 @@ +export type PromptHandle = { + readonly sessionID: string + readonly promptID: string +} + +export function promptKey(handle: PromptHandle) { + return `${handle.sessionID}:${handle.promptID}` +} diff --git a/packages/voice/src/protocol-live.ts b/packages/voice/src/protocol-live.ts new file mode 100644 index 0000000000..23f827aa75 --- /dev/null +++ b/packages/voice/src/protocol-live.ts @@ -0,0 +1,486 @@ +import type { + VoiceConnection, + VoiceDelegationRequest, + VoiceNotificationRequest, + VoiceProtocol, + VoiceProtocolEvent, + VoiceProtocolOptions, +} from "./protocol" +import { decodeVoiceToolInput } from "./protocol" +import { Option, Schema } from "effect" + +const LiveItem = Schema.Struct({ + id: Schema.String, + type: Schema.String, + text: Schema.optional(Schema.String), + name: Schema.optional(Schema.String), + call_id: Schema.optional(Schema.String), + arguments: Schema.optional(Schema.String), + target: Schema.optional(Schema.String), + content: Schema.optional(Schema.Array(Schema.Struct({ type: Schema.String, text: Schema.optional(Schema.String) }))), +}) +const LiveTurn = Schema.Struct({ + id: Schema.String, + role: Schema.Literals(["user", "assistant"]), + transcript: Schema.String, +}) +const LiveEvent = Schema.Struct({ + type: Schema.String, + event_id: Schema.optional(Schema.String), + audio: Schema.optional(Schema.String), + delta: Schema.optional(Schema.String), + start_ms: Schema.optional(Schema.Number), + end_ms: Schema.optional(Schema.Number), + turn_id: Schema.optional(Schema.String), + turn: Schema.optional(LiveTurn), + item: Schema.optional(LiveItem), + error: Schema.optional( + Schema.Struct({ + code: Schema.optional(Schema.String), + message: Schema.optional(Schema.String), + event_id: Schema.optional(Schema.String), + }), + ), +}) +const decodeLiveEvent = Schema.decodeUnknownOption(Schema.fromJsonString(LiveEvent)) +type LiveTurn = Schema.Schema.Type +type LiveEvent = Schema.Schema.Type + +type LiveNotificationResult = { + readonly type?: string + readonly eventID?: string + readonly errorEventID?: string +} + +export function liveNotificationAcknowledgement( + result: LiveNotificationResult, + pending: { + readonly eventID: string + readonly acknowledgement: "session.context.appended" | "delegation.context.appended" + }, +) { + // Live success events do not reliably echo the client event ID. Context + // appends are serialized, so the expected acknowledgement kind is the + // strongest correlation the protocol currently provides. + if (result.type === pending.acknowledgement) return true + if (result.type === "error" && result.errorEventID === pending.eventID) return false + return undefined +} + +type ProjectedTurn = LiveTurn & { readonly displayID: string } + +export function createLiveEventProjector() { + const turns = new Map() + let input: { readonly id: string; readonly transcript: string } | undefined + let assistantTranscript = "" + const userTranscripts = new Map() + const startedTools = new Set() + + const startTool = (item: LiveEvent["item"], events: VoiceProtocolEvent[]) => { + if (item?.type !== "function_call" || !item.name || !item.call_id || startedTools.has(item.call_id)) return + startedTools.add(item.call_id) + events.push({ type: "tool.started", id: item.call_id, name: item.name }) + } + + const syncUser = (id: string, text: string, final: boolean, events: VoiceProtocolEvent[]) => { + text = text.trimStart() + const previous = userTranscripts.get(id) + if (previous === text && !final) return + if (final) userTranscripts.delete(id) + else userTranscripts.set(id, text) + events.push({ type: "user.transcript", id, text, final }) + } + + // Deltas are emitted verbatim, matching the Realtime adapter: a fragment's leading space is + // the only record of the boundary between two assistant turns, so trimming it here would + // weld the last word of one turn onto the first word of the next. Stripping the leading + // edge of a rendered message is the consumer's job. + const syncAssistant = (transcript: string, events: VoiceProtocolEvent[]) => { + if (transcript.startsWith(assistantTranscript)) { + const delta = transcript.slice(assistantTranscript.length) + assistantTranscript = transcript + if (delta) events.push({ type: "assistant.transcript.delta", delta }) + return + } + if (assistantTranscript.startsWith(transcript)) { + assistantTranscript = transcript + events.push({ type: "assistant.transcript", text: transcript }) + return + } + assistantTranscript = transcript + events.push({ type: "assistant.transcript", text: transcript }) + } + + return (message: string) => { + const data = Option.getOrUndefined(decodeLiveEvent(message)) + if (!data) + return { + type: undefined, + events: [{ type: "error", message: "Received an invalid Live API event." }] satisfies VoiceProtocolEvent[], + } + + const events: VoiceProtocolEvent[] = data.type.endsWith(".delta") ? [] : [{ type: "debug", message: data.type }] + switch (data.type) { + case "session.started": + events.push({ type: "ready" }) + break + case "output_audio.delta": + if (data.audio && data.start_ms !== undefined && data.end_ms !== undefined) + events.push({ + type: "assistant.audio", + audio: Buffer.from(data.audio, "base64"), + timeline: { startMs: data.start_ms, endMs: data.end_ms }, + }) + break + case "input_transcript.added": + if (data.item?.type !== "input_transcript" || data.item.text === undefined) break + if (!input) { + input = { id: data.item.id, transcript: "" } + events.push({ type: "user.committed", id: input.id }) + } + input = { ...input, transcript: input.transcript + data.item.text } + syncUser(input.id, input.transcript, false, events) + break + case "output_transcript.added": + if (data.item?.type !== "output_transcript" || data.item.text === undefined) break + const delta = data.item.text + assistantTranscript += delta + if (delta) events.push({ type: "assistant.transcript.delta", delta }) + break + case "turn.created": { + const turn = data.turn + if (!turn) break + const displayID = turn.role === "user" ? (input?.id ?? turn.id) : turn.id + turns.set(turn.id, { ...turn, displayID }) + if (turn.role === "user") { + if (!input) events.push({ type: "user.committed", id: displayID }) + syncUser(displayID, turn.transcript, false, events) + break + } + syncAssistant(turn.transcript, events) + break + } + case "turn.delta": { + if (!data.turn_id || !data.delta) break + const turn = turns.get(data.turn_id) + if (!turn) break + const transcript = turn.transcript + data.delta + turns.set(data.turn_id, { ...turn, transcript }) + if (turn.role === "user") { + syncUser(turn.displayID, transcript, false, events) + break + } + syncAssistant(transcript, events) + break + } + case "turn.done": { + const turn = data.turn + if (!turn) break + const previous = turns.get(turn.id) + if (turn.role === "user") { + const displayID = previous?.displayID ?? input?.id ?? turn.id + if (!previous && !input) events.push({ type: "user.committed", id: displayID }) + syncUser(displayID, turn.transcript, true, events) + input = undefined + turns.delete(turn.id) + break + } + syncAssistant(turn.transcript, events) + assistantTranscript = "" + turns.delete(turn.id) + events.push({ type: "assistant.done", awaitingWork: false }) + break + } + case "response.output_item.added": + startTool(data.item, events) + break + case "delegation.created": + if (data.item?.type !== "delegation" || data.item.target !== "client") break + events.push({ + type: "delegation.requested", + request: { + id: data.item.id, + text: + data.item.content + ?.flatMap((part) => (part.type === "input_text" && part.text ? [part.text] : [])) + .join("\n") ?? "", + }, + }) + break + case "response.output_item.done": { + if (data.item?.type !== "function_call" || !data.item.call_id) break + const name = data.item.name ?? "tool" + startTool(data.item, events) + if (!data.item.name || data.item.arguments === undefined) { + const output = { status: "error", message: "Malformed Live function call." } + events.push({ + type: "work.rejected", + request: { id: data.item.call_id, name, input: {} }, + output, + }) + startedTools.delete(data.item.call_id) + break + } + const input = Option.getOrUndefined(decodeVoiceToolInput(data.item.arguments)) + if (!input) { + const output = { status: "error", message: `Invalid arguments for tool ${data.item.name}.` } + events.push({ type: "error", message: `Received invalid arguments for Live tool ${data.item.name}.` }) + events.push({ + type: "work.rejected", + request: { id: data.item.call_id, name, input: {} }, + output, + }) + startedTools.delete(data.item.call_id) + break + } + events.push({ + type: "work.requested", + request: { id: data.item.call_id, name: data.item.name, input }, + }) + startedTools.delete(data.item.call_id) + break + } + case "error": + events.push({ type: "error", message: `${data.error?.code}: ${data.error?.message}` }) + } + return { + type: data.type, + eventID: data.event_id, + errorEventID: data.error?.event_id, + events, + } + } +} + +export function createLiveProtocol(): VoiceProtocol { + return { + name: "live", + inputActivity: "local", + capabilities: { textInput: false, interruption: false, delegation: true }, + connect: connectLive, + } +} + +export function createLiveEventDelivery(options: { + readonly emit: (event: VoiceProtocolEvent) => void + readonly drainMs?: number +}) { + let pendingDone: Extract | undefined + let timer: ReturnType | undefined + + const schedule = () => { + if (timer) clearTimeout(timer) + timer = setTimeout(() => { + timer = undefined + const event = pendingDone + pendingDone = undefined + if (event) options.emit(event) + }, options.drainMs ?? 120) + } + + return { + push(event: VoiceProtocolEvent) { + if (event.type === "assistant.done") { + pendingDone = event + schedule() + return + } + if (event.type === "assistant.audio" && pendingDone) schedule() + options.emit(event) + }, + close() { + if (timer) clearTimeout(timer) + timer = undefined + pendingDone = undefined + }, + } +} + +export function createLiveContextAppendQueue(options: { + readonly send: (event: Record) => boolean + readonly timeoutMs?: number +}) { + type Item = { + readonly event: Record + readonly acknowledgement: "session.context.appended" | "delegation.context.appended" + readonly result: PromiseWithResolvers + } + const queued: Item[] = [] + let active: (Item & { readonly timer: ReturnType }) | undefined + let closed = false + + const settle = (accepted: boolean) => { + if (!active) return + clearTimeout(active.timer) + active.result.resolve(accepted) + active = undefined + pump() + } + + const pump = () => { + if (closed || active) return + const item = queued.shift() + if (!item) return + active = { + ...item, + timer: setTimeout(() => settle(false), options.timeoutMs ?? 5_000), + } + if (!options.send(item.event)) settle(false) + } + + return { + append( + event: Record, + acknowledgement: "session.context.appended" | "delegation.context.appended", + ) { + if (closed) return Promise.resolve(false) + const result = Promise.withResolvers() + queued.push({ event, acknowledgement, result }) + pump() + return result.promise + }, + receive(result: LiveNotificationResult) { + if (!active) return + const eventID = active.event["event_id"] + const accepted = liveNotificationAcknowledgement(result, { + eventID: typeof eventID === "string" ? eventID : "", + acknowledgement: active.acknowledgement, + }) + if (accepted !== undefined) settle(accepted) + }, + close() { + closed = true + if (active) { + clearTimeout(active.timer) + active.result.resolve(false) + active = undefined + } + queued.splice(0).forEach((item) => item.result.resolve(false)) + }, + } +} + +function connectLive(options: VoiceProtocolOptions): VoiceConnection { + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload. + const ws = new WebSocket(`wss://api.openai.com/v1/live?model=${options.model}`, { + headers: { + Authorization: `Bearer ${options.apiKey}`, + "OpenAI-Alpha": "quicksilver=v2", + }, + } as unknown as string[]) + const closed = Promise.withResolvers() + let closeTimer: ReturnType | undefined + const project = createLiveEventProjector() + const delivery = createLiveEventDelivery({ emit: options.onEvent }) + + const send = (event: Record) => { + if (options.debug || event["type"] !== "input_audio.append") + options.trace?.("live.send", { + type: event["type"], + delegationID: event["delegation_item_id"], + }) + if (ws.readyState !== WebSocket.OPEN) return false + ws.send(JSON.stringify(event)) + return true + } + const context = createLiveContextAppendQueue({ send }) + + const appendNotification = (request: VoiceNotificationRequest) => + context.append( + { + type: "session.context.append", + event_id: crypto.randomUUID(), + content: [{ type: "input_text", text: notificationText(request.text) }], + }, + "session.context.appended", + ) + + ws.addEventListener("open", () => { + send({ + type: "session.update", + event_id: crypto.randomUUID(), + session: { + instructions: options.instructions, + audio: { output: { voice: options.voice } }, + delegation: { type: "client" }, + }, + }) + }) + ws.addEventListener("message", (event) => { + const result = project(String(event.data)) + if (options.debug || result.type !== "output_audio.delta") options.trace?.("live.receive", { type: result.type }) + context.receive(result) + result.events.forEach((event) => { + if (event.type === "assistant.audio") + options.trace?.("live.audio.received", { + bytes: event.audio.length, + startMs: event.timeline?.startMs, + endMs: event.timeline?.endMs, + }) + delivery.push(event) + }) + if (result.type === "session.closed") ws.close(1000) + }) + ws.addEventListener("close", (event) => { + if (closeTimer) clearTimeout(closeTimer) + context.close() + delivery.close() + options.onEvent({ type: "closed", code: event.code }) + closed.resolve() + }) + + return { + appendAudio(audio) { + if (ws.bufferedAmount > 96_000) { + options.trace?.("live.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount }) + return + } + send({ type: "input_audio.append", audio: audio.toString("base64") }) + }, + resolveWork(request, output) { + send({ + type: "delegation.function_call_output.create", + event_id: crypto.randomUUID(), + item: { type: "function_call_output", call_id: request.id, output: JSON.stringify(output) }, + }) + }, + resolveDelegation(request: VoiceDelegationRequest, output: string) { + void context + .append(delegationContext(crypto.randomUUID(), request.id, output), "delegation.context.appended") + .then((accepted) => { + if (!accepted) options.trace?.("live.delegation.context.failed", { delegationID: request.id }) + }) + }, + notify(request) { + return appendNotification(request) + }, + close(closeOptions) { + if (ws.readyState === WebSocket.CLOSED) return Promise.resolve() + if (closeOptions?.graceful === false || ws.readyState !== WebSocket.OPEN) { + ws.close(1000) + closeTimer = setTimeout(() => closed.resolve(), 5_000) + return closed.promise + } + send({ type: "session.close", event_id: crypto.randomUUID() }) + closeTimer = setTimeout(() => { + ws.close(1000) + closed.resolve() + }, 10_500) + return closed.promise + }, + } +} + +function delegationContext(eventID: string, delegationID: string, text: string) { + return { + type: "delegation.context.append", + event_id: eventID, + delegation_item_id: delegationID, + content: [{ type: "input_text", text: notificationText(text) }], + } +} + +function notificationText(output: unknown) { + const text = String(output) + return text.length > 1_600 ? text.slice(0, 1_600) + "..." : text +} diff --git a/packages/voice/src/protocol-realtime.ts b/packages/voice/src/protocol-realtime.ts new file mode 100644 index 0000000000..c92870fb17 --- /dev/null +++ b/packages/voice/src/protocol-realtime.ts @@ -0,0 +1,318 @@ +import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "./protocol" +import { decodeVoiceToolInput } from "./protocol" +import { Option, Schema } from "effect" +import { createSingleFlightAcknowledgement } from "./single-flight-acknowledgement" + +const RealtimeItem = Schema.Struct({ + id: Schema.optional(Schema.String), + type: Schema.optional(Schema.String), + name: Schema.optional(Schema.String), + call_id: Schema.optional(Schema.String), + arguments: Schema.optional(Schema.String), +}) +const RealtimeFunctionCall = Schema.Struct({ + type: Schema.Literal("function_call"), + name: Schema.String, + call_id: Schema.String, + arguments: Schema.String, +}) +const RealtimeEvent = Schema.Struct({ + type: Schema.String, + delta: Schema.optional(Schema.String), + transcript: Schema.optional(Schema.String), + item_id: Schema.optional(Schema.String), + item: Schema.optional(RealtimeItem), + response: Schema.optional(Schema.Struct({ output: Schema.optional(Schema.Array(RealtimeItem)) })), + error: Schema.optional( + Schema.Struct({ + code: Schema.optional(Schema.String), + message: Schema.optional(Schema.String), + event_id: Schema.optional(Schema.String), + }), + ), +}) +const decodeRealtimeEvent = Schema.decodeUnknownOption(Schema.fromJsonString(RealtimeEvent)) +const decodeFunctionCall = Schema.decodeUnknownOption(RealtimeFunctionCall) +type RealtimeEvent = Schema.Schema.Type +type RealtimeProjectorCommand = { readonly type: "response.create" } + +export function createRealtimeEventProjector() { + const pendingCalls = new Set() + const resolvedCalls = new Set() + const startedCalls = new Set() + const projectedCalls = new Set() + let responseAwaitingWork = false + + const resumeAfterWork = (): ReadonlyArray => { + if (!responseAwaitingWork || pendingCalls.size > 0) return [] + responseAwaitingWork = false + resolvedCalls.clear() + return [{ type: "response.create" }] + } + + return { + receive(raw: string) { + const event = Option.getOrUndefined(decodeRealtimeEvent(raw)) + if (!event) + return { + type: "invalid", + events: [{ type: "error", message: "Received an invalid Realtime API event." }] satisfies VoiceProtocolEvent[], + commands: [] as ReadonlyArray, + } + const events: VoiceProtocolEvent[] = event.type.endsWith(".delta") + ? [] + : [{ type: "debug", message: event.type }] + const commands: RealtimeProjectorCommand[] = [] + switch (event.type) { + case "session.created": + events.push({ type: "ready" }) + break + case "response.output_text.delta": + case "response.output_audio_transcript.delta": + events.push({ type: "assistant.transcript.delta", delta: event.delta ?? "" }) + break + case "response.done": { + const callIDs = (event.response?.output ?? []).flatMap((item) => + item.type === "function_call" && item.call_id ? [item.call_id] : [], + ) + callIDs.filter((id) => !resolvedCalls.has(id)).forEach((id) => pendingCalls.add(id)) + responseAwaitingWork = callIDs.length > 0 + events.push({ type: "assistant.done", awaitingWork: responseAwaitingWork }) + commands.push(...resumeAfterWork()) + break + } + case "input_audio_buffer.speech_started": + events.push({ type: "user.started" }) + break + case "input_audio_buffer.speech_stopped": + events.push({ type: "user.stopped" }) + break + case "input_audio_buffer.committed": + events.push({ type: "user.committed", id: event.item_id ?? "" }) + break + case "conversation.item.input_audio_transcription.completed": + events.push({ + type: "user.transcript", + id: event.item_id ?? "", + text: (event.transcript ?? "").trim(), + final: true, + }) + break + case "response.output_audio.delta": + if (event.delta) events.push({ type: "assistant.audio", audio: Buffer.from(event.delta, "base64") }) + break + case "response.output_item.added": + if ( + event.item?.type === "function_call" && + event.item.name && + event.item.call_id && + !startedCalls.has(event.item.call_id) + ) { + startedCalls.add(event.item.call_id) + events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name }) + } + break + case "response.output_item.done": { + if (event.item?.type !== "function_call") break + if (event.item.call_id && projectedCalls.has(event.item.call_id)) break + if (event.item.name && event.item.call_id && !startedCalls.has(event.item.call_id)) { + startedCalls.add(event.item.call_id) + events.push({ type: "tool.started", id: event.item.call_id, name: event.item.name }) + } + const call = Option.getOrUndefined(decodeFunctionCall(event.item)) + if (!call) { + events.push({ type: "error", message: "Received a malformed Realtime function call." }) + if (event.item.call_id) + events.push({ + type: "work.rejected", + request: { id: event.item.call_id, name: event.item.name ?? "tool", input: {} }, + output: { status: "error", message: "Malformed Realtime function call." }, + }) + if (event.item.call_id) startedCalls.delete(event.item.call_id) + if (event.item.call_id) projectedCalls.add(event.item.call_id) + break + } + pendingCalls.add(call.call_id) + const input = Option.getOrUndefined(decodeVoiceToolInput(call.arguments)) + if (!input) { + events.push({ type: "error", message: `Received invalid arguments for Realtime tool ${call.name}.` }) + events.push({ + type: "work.rejected", + request: { id: call.call_id, name: call.name, input: {} }, + output: { status: "error", message: `Invalid arguments for tool ${call.name}.` }, + }) + startedCalls.delete(call.call_id) + projectedCalls.add(call.call_id) + break + } + events.push({ type: "work.requested", request: { id: call.call_id, name: call.name, input } }) + startedCalls.delete(call.call_id) + projectedCalls.add(call.call_id) + break + } + case "error": + events.push({ type: "error", message: `${event.error?.code}: ${event.error?.message}` }) + } + return { type: event.type, event, events, commands } + }, + resolveWork(id: string) { + resolvedCalls.add(id) + pendingCalls.delete(id) + return resumeAfterWork() + }, + } +} + +export function realtimeNotificationAcknowledgement( + event: Pick, + pending: { readonly itemID: string; readonly eventID: string }, +) { + if (event.type === "conversation.item.created" && event.item?.id === pending.itemID) return true + if (event.type === "error" && event.error?.event_id === pending.eventID) return false + return undefined +} + +export function createRealtimeProtocol(): VoiceProtocol { + return { + name: "realtime", + inputActivity: "server", + capabilities: { textInput: true, interruption: true, delegation: false }, + connect: connectRealtime, + } +} + +export function realtimeSessionUpdate( + options: Pick, +) { + return { + type: "session.update", + session: { + type: "realtime", + instructions: options.instructions, + tools: options.tools, + tool_choice: "auto", + audio: { output: { voice: options.voice } }, + }, + } +} + +function connectRealtime(options: VoiceProtocolOptions): VoiceConnection { + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion -- Bun accepts WebSocket headers in this runtime-only overload. + const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${options.model}`, { + headers: { Authorization: `Bearer ${options.apiKey}` }, + } as unknown as string[]) + const closed = Promise.withResolvers() + let closeTimer: ReturnType | undefined + const notification = createSingleFlightAcknowledgement<{ readonly itemID: string; readonly eventID: string }>() + const projector = createRealtimeEventProjector() + + const send = (event: Record) => { + if (options.debug || event["type"] !== "input_audio_buffer.append") + options.trace?.("realtime.send", { + type: event["type"], + callID: + event["item"] && typeof event["item"] === "object" && "call_id" in event["item"] + ? event["item"].call_id + : undefined, + }) + if (ws.readyState !== WebSocket.OPEN) return false + ws.send(JSON.stringify(event)) + return true + } + const createResponse = (text = false) => + send(text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" }) + const resolveFunctionCall = (id: string, output: unknown) => { + send({ + type: "conversation.item.create", + item: { type: "function_call_output", call_id: id, output: JSON.stringify(output) }, + }) + projector.resolveWork(id).forEach(() => createResponse()) + } + + ws.addEventListener("open", () => { + send(realtimeSessionUpdate(options)) + send({ + type: "session.update", + session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } }, + }) + send({ + type: "session.update", + session: { + type: "realtime", + audio: { + input: { + turn_detection: { + type: "server_vad", + silence_duration_ms: 900, + interrupt_response: options.fullDuplex, + }, + }, + }, + }, + }) + }) + ws.addEventListener("message", (event) => { + const result = projector.receive(String(event.data)) + if (options.debug || result.type !== "response.output_audio.delta") + options.trace?.("realtime.receive", { type: result.type }) + const current = notification.current() + const acknowledgement = current && result.event + ? realtimeNotificationAcknowledgement(result.event, current.correlation) + : undefined + if (acknowledgement !== undefined && current) notification.settle(current.id, acknowledgement) + result.events.forEach(options.onEvent) + result.commands.forEach(() => createResponse()) + }) + ws.addEventListener("close", (event) => { + if (closeTimer) clearTimeout(closeTimer) + notification.close() + options.onEvent({ type: "closed", code: event.code }) + closed.resolve() + }) + + return { + appendAudio(audio) { + if (ws.bufferedAmount > 96_000) { + options.trace?.("realtime.audio.dropped", { bytes: audio.length, buffered: ws.bufferedAmount }) + return + } + send({ type: "input_audio_buffer.append", audio: audio.toString("base64") }) + }, + sendText(text) { + send({ + type: "conversation.item.create", + item: { type: "message", role: "user", content: [{ type: "input_text", text }] }, + }) + createResponse(true) + }, + resolveWork(request, output) { + resolveFunctionCall(request.id, output) + }, + notify(request) { + const suffix = crypto.randomUUID().replaceAll("-", "") + const itemID = `item_${suffix}` + const eventID = `event_${suffix}` + const pending = notification.begin(request.id, { itemID, eventID }) + if (!pending.started) return pending.promise + const created = send({ + type: "conversation.item.create", + event_id: eventID, + item: { id: itemID, type: "message", role: "user", content: [{ type: "input_text", text: request.text }] }, + }) + if (!created || !createResponse()) { + notification.settle(request.id, false) + return Promise.resolve(false) + } + return pending.promise + }, + interrupt() { + send({ type: "response.cancel" }) + }, + close() { + if (ws.readyState === WebSocket.CLOSED) return Promise.resolve() + ws.close(1000) + closeTimer ??= setTimeout(() => closed.resolve(), 5_000) + return closed.promise + }, + } +} diff --git a/packages/voice/src/protocol.ts b/packages/voice/src/protocol.ts new file mode 100644 index 0000000000..c86c4e46b5 --- /dev/null +++ b/packages/voice/src/protocol.ts @@ -0,0 +1,94 @@ +import { Schema } from "effect" + +export const decodeVoiceToolInput = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +) + +export type VoiceTool = { + readonly type: "function" + readonly name: string + readonly description: string + readonly parameters: unknown +} + +export type VoiceWorkRequest = { + readonly id: string + readonly name: string + readonly input: Record +} + +export type VoiceToolExecution = { + readonly output: unknown + readonly admittedPrompt?: { + readonly sessionID: string + readonly promptID: string + } +} + +export type VoiceDelegationRequest = { + readonly id: string + readonly text: string +} + +export type VoiceNotificationRequest = { + readonly id: string + readonly text: string + readonly delegationID?: string +} + +export type VoiceAudioTimeline = { readonly startMs: number; readonly endMs: number } + +export type VoiceProtocolEvent = + | { readonly type: "ready" } + | { readonly type: "user.started" } + | { readonly type: "user.stopped" } + | { readonly type: "user.committed"; readonly id: string } + | { readonly type: "user.transcript"; readonly id: string; readonly text: string; readonly final: boolean } + | { + readonly type: "assistant.audio" + readonly audio: Buffer + readonly timeline?: VoiceAudioTimeline + } + | { readonly type: "assistant.transcript.delta"; readonly delta: string } + | { readonly type: "assistant.transcript"; readonly text: string } + | { readonly type: "assistant.done"; readonly awaitingWork: boolean } + | { readonly type: "tool.started"; readonly id: string; readonly name: string } + | { readonly type: "work.requested"; readonly request: VoiceWorkRequest } + | { readonly type: "work.rejected"; readonly request: VoiceWorkRequest; readonly output: unknown } + | { readonly type: "delegation.requested"; readonly request: VoiceDelegationRequest } + | { readonly type: "debug"; readonly message: string } + | { readonly type: "error"; readonly message: string } + | { readonly type: "closed"; readonly code: number } + +export type VoiceProtocolOptions = { + readonly apiKey: string + readonly model: string + readonly voice: string + readonly instructions: string + readonly tools: ReadonlyArray + readonly fullDuplex: boolean + readonly debug: boolean + readonly onEvent: (event: VoiceProtocolEvent) => void + readonly trace?: (event: string, data?: Record) => void +} + +export type VoiceConnection = { + appendAudio(audio: Buffer): void + sendText?(text: string): void + resolveWork(request: VoiceWorkRequest, output: unknown): void + resolveDelegation?(request: VoiceDelegationRequest, output: string): void + notify(notification: VoiceNotificationRequest): Promise + interrupt?(): void + close(options?: { readonly graceful?: boolean }): Promise +} + +export type VoiceProtocol = { + readonly name: "live" | "realtime" + readonly inputActivity: "local" | "server" + readonly capabilities: { + readonly textInput: boolean + readonly interruption: boolean + readonly delegation: boolean + } + connect(options: VoiceProtocolOptions): VoiceConnection +} diff --git a/packages/voice/src/responses-controller.ts b/packages/voice/src/responses-controller.ts new file mode 100644 index 0000000000..b221e9e2da --- /dev/null +++ b/packages/voice/src/responses-controller.ts @@ -0,0 +1,167 @@ +import { Option } from "effect" +import { decodeVoiceToolInput, type VoiceTool, type VoiceWorkRequest } from "./protocol" + +export type ResponsesControllerContext = { + readonly sessionIDs: Set + lastSessionID?: string +} + +export function createResponsesControllerContext(): ResponsesControllerContext { + return { sessionIDs: new Set() } +} + +export function responsesControllerTools( + tools: ReadonlyArray, + text: string, + context: ResponsesControllerContext, +) { + if (!context.lastSessionID || /\b(new|another|separate|fresh)\s+(session|thread)\b/i.test(text)) return tools + return tools.filter((tool) => tool.name !== "start_session") +} + +export async function runResponsesController(options: { + apiKey: string + model: string + instructions: string + text: string + tools: ReadonlyArray + execute: (call: VoiceWorkRequest) => Promise + context?: ResponsesControllerContext + trace?: (event: string, data?: Record) => void + fetch?: typeof fetch +}) { + const request = options.fetch ?? fetch + const context = options.context ?? createResponsesControllerContext() + const cache = new Map>() + let input: unknown = controllerInput(options.text, context) + let previousResponseID: string | undefined + + for (let step = 0; step < 20; step++) { + options.trace?.("responses.controller.requested", { step, continued: previousResponseID !== undefined }) + const response = await request("https://api.openai.com/v1/responses", { + method: "POST", + headers: { Authorization: `Bearer ${options.apiKey}`, "Content-Type": "application/json" }, + body: JSON.stringify({ + model: options.model, + instructions: options.instructions, + input, + previous_response_id: previousResponseID, + tools: options.tools, + store: true, + }), + }) + if (!response.ok) throw new Error(`Responses controller failed (${response.status}): ${await response.text()}`) + const value = requireResponse(await response.json()) + const calls = value.output.flatMap((item) => { + if ( + item.type !== "function_call" || + typeof item.call_id !== "string" || + typeof item.name !== "string" || + typeof item.arguments !== "string" + ) + return [] + const parsed = Option.getOrUndefined(decodeVoiceToolInput(item.arguments)) + if (!parsed) throw new Error(`Responses controller returned invalid arguments for ${item.name}.`) + return [{ id: item.call_id, name: item.name, input: parsed }] + }) + options.trace?.("responses.controller.responded", { + step, + responseID: value.id, + tools: calls.map((call) => call.name), + }) + if (calls.length === 0) + return withSessionContext( + responseText(value.output) || "The controller finished without a text reply.", + context.sessionIDs, + ) + input = await Promise.all( + calls.map(async (call) => { + if (typeof call.input["session_id"] === "string") rememberSession(context, call.input["session_id"]) + const key = `${call.name}:${JSON.stringify(call.input)}` + const cached = cacheableTools.has(call.name) ? cache.get(key) : undefined + if (cached) options.trace?.("responses.controller.tool.reused", { step, callID: call.id, name: call.name }) + const execution = + cached ?? + Promise.resolve().then(() => { + options.trace?.("responses.controller.tool.started", { step, callID: call.id, name: call.name }) + return options.execute(call) + }) + if (!cached && cacheableTools.has(call.name)) cache.set(key, execution) + const output = await execution + rememberOutputSessions(context, output) + if (!cached) options.trace?.("responses.controller.tool.resolved", { step, callID: call.id, name: call.name }) + return { + type: "function_call_output", + call_id: call.id, + output: JSON.stringify(output), + } + }), + ) + previousResponseID = value.id + } + throw new Error("Responses controller exceeded its tool-call limit.") +} + +const cacheableTools = new Set([ + "find_projects", + "find_sessions", + "read_session", + "list_pending_permissions", + "list_pending_questions", + "list_pending_forms", +]) + +function rememberSession(context: ResponsesControllerContext, sessionID: string) { + context.sessionIDs.add(sessionID) + context.lastSessionID = sessionID +} + +function rememberOutputSessions(context: ResponsesControllerContext, output: unknown) { + if (!output || typeof output !== "object") return + if ("session_id" in output && typeof output.session_id === "string") rememberSession(context, output.session_id) + if (!("sessions" in output) || !Array.isArray(output.sessions)) return + const sessionIDs = output.sessions.flatMap((session) => + session && typeof session === "object" && "id" in session && typeof session.id === "string" ? [session.id] : [], + ) + sessionIDs.forEach((sessionID) => context.sessionIDs.add(sessionID)) + if (sessionIDs.length === 1) context.lastSessionID = sessionIDs[0] +} + +function controllerInput(text: string, context: ResponsesControllerContext) { + if (!context.lastSessionID) return text + return `${text}\n\nPrivate voice-control context (never mention or expose this): the most recently used OpenCode session ID is ${context.lastSessionID}. Resolve references like "that session", "it", or "stop that" directly against this ID without searching again.` +} + +function withSessionContext(text: string, sessionIDs: ReadonlySet) { + if (sessionIDs.size === 0) return text + return `${text}\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ${[...sessionIDs].join(", ")}. Include the relevant session_id in any future delegated request to continue this work.` +} + +function requireResponse(value: unknown) { + if (!value || typeof value !== "object" || !("id" in value) || typeof value.id !== "string" || !("output" in value)) + throw new Error("Responses controller returned an invalid response.") + if (!Array.isArray(value.output)) throw new Error("Responses controller returned invalid output.") + return { + id: value.id, + output: value.output.filter( + (item): item is Record & { readonly type: string } => + !!item && typeof item === "object" && "type" in item && typeof item.type === "string", + ), + } +} + +function responseText(output: ReadonlyArray>) { + return output + .flatMap((item) => (Array.isArray(item["content"]) ? item["content"] : [])) + .flatMap((part) => + part && + typeof part === "object" && + "type" in part && + part.type === "output_text" && + "text" in part && + typeof part.text === "string" + ? [part.text] + : [], + ) + .join("\n") +} diff --git a/packages/voice/src/single-flight-acknowledgement.ts b/packages/voice/src/single-flight-acknowledgement.ts new file mode 100644 index 0000000000..1d666a4782 --- /dev/null +++ b/packages/voice/src/single-flight-acknowledgement.ts @@ -0,0 +1,38 @@ +export function createSingleFlightAcknowledgement(timeoutMs = 5_000) { + let pending: + | { + readonly id: string + readonly correlation: Correlation + readonly result: PromiseWithResolvers + readonly timer: ReturnType + } + | undefined + + const settle = (id: string, accepted: boolean) => { + if (pending?.id !== id) return + clearTimeout(pending.timer) + pending.result.resolve(accepted) + pending = undefined + } + + return { + begin(id: string, correlation: Correlation) { + if (pending) return { started: false, promise: Promise.resolve(false) } as const + const result = Promise.withResolvers() + pending = { + id, + correlation, + result, + timer: setTimeout(() => settle(id, false), timeoutMs), + } + return { started: true, promise: result.promise } as const + }, + current() { + return pending ? { id: pending.id, correlation: pending.correlation } : undefined + }, + settle, + close() { + if (pending) settle(pending.id, false) + }, + } +} diff --git a/packages/voice/src/spike.ts b/packages/voice/src/spike.ts index 084a6dc790..c0892b1727 100644 --- a/packages/voice/src/spike.ts +++ b/packages/voice/src/spike.ts @@ -1,31 +1,38 @@ #!/usr/bin/env bun -// Voice control spike: bridges the local microphone and speaker to the OpenAI -// Realtime API (gpt-realtime-2.1) and exposes OpenCode session control as -// realtime function tools, so you can drive OpenCode with your voice. +// Voice control spike: bridges the local microphone and speaker to OpenAI's +// Realtime or Live voice API and delegates coding work to OpenCode. // // Usage: -// opencode serve (or note the URL of a running server) -// 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- \ -// bun packages/voice/src/spike.ts --server http://localhost:4096 [--directory /path/to/project] +// bun run --cwd packages/voice spike [--backend realtime|live] [--directory /path/to/project] +// Bun loads packages/voice/.env automatically when the command runs from that package. // // Requires sox (`brew install sox`) for mic capture (`rec`) and playback (`play`). import { parseArgs } from "node:util" -import { realpathSync } from "node:fs" -import { tmpdir } from "node:os" import { OpenCode } from "@opencode-ai/client/promise" -import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client/promise" +import { Service } from "@opencode-ai/client/service" +import { createAudioSession, type AudioSession } from "./audio-session" +import type { OpenCodeAnnouncement } from "./opencode-notification" +import { createVoiceTrace } from "./trace" +import { + createResponsesControllerContext, + responsesControllerTools, + runResponsesController, +} from "./responses-controller" +import { createVoiceSession, voiceControlTool, type VoiceSession } from "./voice-session" const args = parseArgs({ options: { + backend: { type: "string", default: "realtime" }, server: { type: "string" }, + password: { type: "string" }, directory: { type: "string", default: process.cwd() }, - model: { type: "string", default: "gpt-realtime-2.1" }, + model: { type: "string" }, + "delegation-model": { type: "string", default: "gpt-5.5" }, voice: { type: "string", default: "marin" }, provider: { type: "string", default: "openai" }, "coding-model": { type: "string", default: "gpt-5.6-sol" }, variant: { type: "string", default: "medium" }, - password: { type: "string" }, // Keep the mic hot while the assistant speaks (voice barge-in). Only // usable with headphones: on speakers the mic hears the assistant and // interrupts it with its own echo. Default is half-duplex gating. @@ -37,77 +44,57 @@ const args = parseArgs({ // Text mode: send one typed message instead of opening the microphone, // print the reply, and exit. Useful for smoke-testing the tool loop. text: { type: "string" }, - // Log every realtime event type as it arrives. + // Log every protocol event type as it arrives. debug: { type: "boolean", default: false }, "reduce-motion": { type: "boolean", default: false }, }, }).values +let audioSession: AudioSession | undefined +let voiceSession: VoiceSession | undefined -if (!args.server) { - console.error("Usage: bun src/spike.ts --server http://localhost:PORT [--directory /path]") - console.error("Start a server first with `opencode serve` and pass its URL.") +if (args.backend !== "realtime" && args.backend !== "live") { + console.error("--backend must be realtime or live") process.exit(1) } -const apiKey = process.env["OPENAI_API_KEY"] -if (!apiKey) { - console.error("OPENAI_API_KEY is required. Run via:") - console.error( - ` 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- bun src/spike.ts ...`, - ) +const protocol = + args.backend === "live" + ? (await import("./protocol-live")).createLiveProtocol() + : (await import("./protocol-realtime")).createRealtimeProtocol() +const model = args.model ?? (protocol.name === "live" ? "gpt-live-1-boulder-alpha" : "gpt-realtime-2.1") +if (args.text && !protocol.capabilities.textInput) { + console.error(`--text is not supported by the ${protocol.name} backend`) + process.exit(1) +} +const apiKey = (() => { + const value = process.env["OPENAI_API_KEY"] + if (value) return value + console.error("OPENAI_API_KEY is required. Add it to the gitignored packages/voice/.env or export it in the shell.") + process.exit(1) + return "" +})() +if (!args.text && !process.stdout.isTTY) { + console.error("The voice TUI requires direct terminal access for screen rendering and keyboard input.") + console.error("1Password output masking pipes stdout, so `2password run` cannot host this interactive TUI.") + console.error("Run directly: bun run --cwd packages/voice spike --backend " + protocol.name) process.exit(1) } -const password = args.password ?? process.env["OPENCODE_SERVER_PASSWORD"] +const serverPassword = args.password ?? process.env["OPENCODE_PASSWORD"] ?? process.env["OPENCODE_SERVER_PASSWORD"] +const endpoint = args.server + ? { + url: args.server, + auth: serverPassword ? { type: "basic" as const, username: "opencode", password: serverPassword } : undefined, + } + : ((await Service.discover()) ?? (await Service.ensure({ command: ["opencode2", "serve", "--service"] }))) const client = OpenCode.make({ - baseUrl: args.server, - headers: password ? { Authorization: "Basic " + btoa("opencode:" + password) } : undefined, + baseUrl: endpoint.url, + headers: Service.headers(endpoint), }) const health = await client.health.get().catch((error) => { - console.error(`Could not reach the opencode server at ${args.server}: ${error}`) - console.error("If the server printed a password on startup, pass it with --password ") - console.error("or export OPENCODE_SERVER_PASSWORD.") + console.error(`Could not reach the OpenCode 2 server at ${endpoint.url}: ${error}`) process.exit(1) }) -// --------------------------------------------------------------------------- -// OpenCode tool surface exposed to the realtime model -// --------------------------------------------------------------------------- - -let activeSessionID: string | undefined -let activeDirectory = args.directory! -let activeProjectID: string | undefined -let lastPromptAt = 0 -const selectableProjectIDs = new Set() -const selectableSessionIDs = new Set() - -function assistantText(message: SessionMessageAssistant) { - return message.content - .filter((part) => part.type === "text") - .map((part) => part.text) - .join("\n") -} - -function latestAssistant(messages: ReadonlyArray) { - return messages.find((message): message is SessionMessageAssistant => message.type === "assistant") -} - -async function requireSession() { - if (activeSessionID) return activeSessionID - const created = await client.session.create({ - location: { directory: activeDirectory }, - model: { - providerID: args.provider, - id: args["coding-model"], - variant: args.variant, - }, - }) - activeSessionID = created.id - ui.meta(`[session] created ${activeSessionID} · ${args["coding-model"]}:${args.variant}`) - ui.setStatus({ session: activeSessionID }) - return created.id -} - -const truncate = (text: string, max = 2000) => (text.length > max ? text.slice(0, max) + "…" : text) - +const trace = await createVoiceTrace() // --------------------------------------------------------------------------- // UI: OpenTUI in voice mode, plain console in --text mode. Created before the // WebSocket so no await sits between socket creation and handler registration. @@ -117,671 +104,164 @@ const { createConsoleUI, createVoiceTUI } = await import("./ui") const tuiActive = !args.text const ui = tuiActive ? await createVoiceTUI({ - onInterrupt: () => interrupt(), + onInterrupt: () => voiceSession?.interrupt(), onExit: () => shutdown(), - onCycleVoice: () => cycleVoice(), + onCycleVoice: () => voiceSession?.cycleVoice(), + onToggleMicrophone: () => voiceSession?.toggleMicrophone(), + onToggleSpeaker: () => voiceSession?.toggleSpeaker(), reducedMotion: args["reduce-motion"], }) : createConsoleUI() -ui.setStatus({ server: args.server, model: `${args["coding-model"]}:${args.variant}` }) -ui.meta(`opencode ${args.server} (version ${health.version})`) -ui.meta(`project ${activeDirectory}`) +ui.setStatus({ server: endpoint.url, model: `${args["coding-model"]}:${args.variant}` }) +ui.meta(`opencode ${endpoint.url} (version ${health.version})`) +ui.meta(`project ${args.directory}`) +ui.meta(`trace ${trace.path}`) +trace.write("voice.started", { + backend: protocol.name, + model, + directory: args.directory, + duplex: args.duplex, + speakers: args.speakers, +}) -const toolError = (message: string, retryable = false) => ({ status: "error", message, retryable }) -const projectLabel = (project: { name?: string; worktree: string }) => - project.name ?? project.worktree.replaceAll("\\", "/").split("/").at(-1) ?? "project" -const listKnownProjects = async () => { - const seen = new Set() - const temporary = realpathSync(tmpdir()) - return (await client.project.list()) - .sort((a, b) => b.time.updated - a.time.updated) - .filter((project) => { - if (project.id === "global" || project.worktree.startsWith(temporary) || seen.has(project.worktree)) return false - seen.add(project.worktree) - return true - }) -} +const voices = ["marin", "cedar", "coral", "sage", "ash", "ballad", "alloy", "verse"] +const pendingNotifications: OpenCodeAnnouncement[] = [] +const { createOpenCodeBridge } = await import("./opencode") +const opencode = await createOpenCodeBridge({ + client, + directory: args.directory, + model: { providerID: args.provider, id: args["coding-model"], variant: args.variant }, + notify: (announcement) => { + if (voiceSession) return voiceSession.queueNotification(announcement) + pendingNotifications.push(announcement) + }, + trace: (event, data) => trace.write(event, data), +}).catch(async (error) => { + ui.close() + await trace.close() + console.error(`Voice startup failed: ${String(error)}`) + process.exit(1) +}) +const toolDefinitions = [...opencode.definitions, voiceControlTool(voices)] -const toolHandlers: Record) => Promise> = { - find_projects: async (input) => { - const query = typeof input["query"] === "string" ? input["query"].toLowerCase() : undefined - const limit = typeof input["limit"] === "number" ? input["limit"] : 10 - const projects = (await listKnownProjects()) - .filter((project) => !query || projectLabel(project).toLowerCase().includes(query)) - .slice(0, limit) - selectableProjectIDs.clear() - projects.forEach((project) => selectableProjectIDs.add(project.id)) - return { - status: "ok", - projects: projects.map((project) => ({ - id: project.id, - name: projectLabel(project), - directories: 1 + project.sandboxes.length, - updated: new Date(project.time.updated).toISOString(), - active: project.id === activeProjectID, - })), - } - }, - select_project: async (input) => { - const projectID = input["project_id"] - if (typeof projectID !== "string" || !selectableProjectIDs.has(projectID)) - return toolError("Select a project ID returned by the latest find_projects call.") - const project = (await listKnownProjects()).find((item) => item.id === projectID) - if (!project) return toolError("That project is no longer available.", true) - activeProjectID = project.id - activeDirectory = project.worktree - activeSessionID = undefined - selectableSessionIDs.clear() - ui.setStatus({ project: projectLabel(project), session: undefined }) - ui.meta(`[project] selected ${projectLabel(project)}`) - return { status: "selected", projectID: project.id, name: projectLabel(project) } - }, - find_sessions: async (input) => { - const query = typeof input["query"] === "string" ? input["query"] : undefined - const scope = input["scope"] === "all_projects" ? "all_projects" : "current_project" - const recency = typeof input["recency"] === "string" ? input["recency"] : "any" - const limit = typeof input["limit"] === "number" ? input["limit"] : 10 - const durations: Record = { day: 86_400_000, week: 604_800_000, month: 2_592_000_000 } - const threshold = recency === "any" ? 0 : Date.now() - (durations[recency] ?? 0) - const result = await client.session.list({ - ...(scope === "current_project" ? { directory: activeDirectory } : {}), - search: query, - parentID: null, - limit: recency === "any" ? limit : Math.min(limit * 5, 100), - order: "desc", - }) - const projects = new Map((await listKnownProjects()).map((project) => [project.id, projectLabel(project)])) - const sessions = result.data - .filter((session) => projects.has(session.projectID) && session.time.updated >= threshold) - .slice(0, limit) - selectableSessionIDs.clear() - sessions.forEach((session) => selectableSessionIDs.add(session.id)) - return { - status: "ok", - scope, - sessions: sessions.map((session) => ({ - id: session.id, - title: session.title, - project: projects.get(session.projectID) ?? "project", - updated: new Date(session.time.updated).toISOString(), - active: session.id === activeSessionID, - })), - } - }, - select_session: async (input) => { - const sessionID = input["session_id"] - if (typeof sessionID !== "string" || !selectableSessionIDs.has(sessionID)) - return toolError("Select a session ID returned by the latest find_sessions call.") - const session = await client.session.get({ sessionID }).catch(() => undefined) - if (!session) return toolError("That session is no longer available.", true) - activeSessionID = session.id - activeDirectory = session.location.directory - activeProjectID = session.projectID - const project = (await listKnownProjects()).find((item) => item.id === session.projectID) - ui.setStatus({ session: session.id, project: project ? projectLabel(project) : undefined }) - return { status: "selected", sessionID: session.id, title: session.title } - }, - start_task: async (input) => { - const text = input["text"] - if (typeof text !== "string" || text.length === 0) return toolError("Task text is required.") - if (input["new_session"] === true) activeSessionID = undefined - const sessionID = await requireSession() - lastPromptAt = Date.now() - await client.session.prompt({ sessionID, text }) - if (input["delivery"] === "background") - return { status: "started", sessionID, hint: "Use check_session only when the user asks for status." } - await client.session.wait({ sessionID }) - const messages = await client.message.list({ sessionID, order: "desc", limit: 20 }) - const assistant = latestAssistant(messages.data) - if (!assistant || assistant.time.created < lastPromptAt) - return { status: "completed", sessionID, text: "The coding agent finished without a text reply." } - return { status: "completed", sessionID, text: truncate(assistantText(assistant)) } - }, - check_session: async () => { - if (!activeSessionID) return toolError("No active session. Find one or start a task first.") - const messages = await client.message.list({ sessionID: activeSessionID, order: "desc", limit: 20 }) - const assistant = latestAssistant(messages.data) - if (!assistant) return { status: "waiting", runningTools: [], text: null } - const tools = assistant.content.filter((part) => part.type === "tool").map((part) => part.name) - return { - status: assistant.time.completed ? "completed" : "working", - runningTools: tools, - text: truncate(assistantText(assistant)), - } - }, - interrupt_session: async () => { - if (!activeSessionID) return toolError("No active session to interrupt.") - await client.session.interrupt({ sessionID: activeSessionID }) - return { status: "interrupted", sessionID: activeSessionID } - }, - list_pending_permissions: async () => { - if (!activeSessionID) return toolError("No active session.") - const requests = await client.permission.list({ sessionID: activeSessionID }) - return { - status: "ok", - requests: requests.map((request) => ({ - id: request.id, - action: request.action, - resources: request.resources, - })), - } - }, - reply_permission: async (input) => { - if (!activeSessionID) return toolError("No active session.") - const requestID = input["request_id"] - const decision = input["decision"] - if (typeof requestID !== "string" || (decision !== "allow_once" && decision !== "reject")) - return toolError("A valid request ID and decision are required.") - const requests = await client.permission.list({ sessionID: activeSessionID }) - if (!requests.some((request) => request.id === requestID)) - return toolError("That permission request is not pending.", true) - await client.permission.reply({ - sessionID: activeSessionID, - requestID, - reply: decision === "allow_once" ? "once" : "reject", - }) - return { status: decision === "allow_once" ? "allowed_once" : "rejected", requestID } - }, - set_voice: async (input) => { - const voice = input["voice"] - if (typeof voice !== "string" || !voices.includes(voice)) - return toolError(`Voice must be one of: ${voices.join(", ")}.`) - setTimeout(() => setVoice(voice), 1000) - return { status: "switching", voice, note: "Realtime conversation memory resets during reconnect." } - }, -} - -const emptyParameters = { type: "object", additionalProperties: false, properties: {}, required: [] } -const toolDefinitions = [ - { - type: "function", - name: "find_projects", - description: - "Find known OpenCode projects by display name. Returns opaque IDs, never filesystem paths. Use before select_project.", - parameters: { - type: "object", - additionalProperties: false, - properties: { - query: { type: ["string", "null"], description: "Name fragment, or null for recent projects." }, - limit: { type: "integer", minimum: 1, maximum: 20 }, - }, - required: ["query", "limit"], - }, - }, - { - type: "function", - name: "select_project", - description: "Select a project returned by find_projects and clear the active session.", - parameters: { - type: "object", - additionalProperties: false, - properties: { project_id: { type: "string", description: "Opaque project ID from find_projects." } }, - required: ["project_id"], - }, - }, - { - type: "function", - name: "find_sessions", - description: - "Find root OpenCode sessions by title and recency. Search the current project by default; use all_projects only when the user asks or current-project search misses.", - parameters: { - type: "object", - additionalProperties: false, - properties: { - query: { type: ["string", "null"], description: "Title words, or null for recent sessions." }, - scope: { type: "string", enum: ["current_project", "all_projects"] }, - recency: { type: "string", enum: ["day", "week", "month", "any"] }, - limit: { type: "integer", minimum: 1, maximum: 20 }, - }, - required: ["query", "scope", "recency", "limit"], - }, - }, - { - type: "function", - name: "select_session", - description: "Select a session returned by the latest find_sessions call.", - parameters: { - type: "object", - additionalProperties: false, - properties: { session_id: { type: "string", description: "Session ID from find_sessions." } }, - required: ["session_id"], - }, - }, - { - type: "function", - name: "start_task", - description: - "Send work to the active coding session, creating one if needed. Use wait for quick questions whose answer should be spoken now; use background for coding work.", - parameters: { - type: "object", - additionalProperties: false, - properties: { - text: { type: "string", description: "Clear instruction for the coding agent." }, - new_session: { type: "boolean", description: "True only when the user explicitly asks for a new session." }, - delivery: { type: "string", enum: ["wait", "background"] }, - }, - required: ["text", "new_session", "delivery"], - }, - }, - { - type: "function", - name: "check_session", - description: "Check active coding-session status only when the user asks for an update.", - parameters: emptyParameters, - }, - { - type: "function", - name: "interrupt_session", - description: "Interrupt active coding work. Call only after the user explicitly confirms the interruption.", - parameters: emptyParameters, - }, - { - type: "function", - name: "list_pending_permissions", - description: "List permission requests blocking the active coding session.", - parameters: emptyParameters, - }, - { - type: "function", - name: "reply_permission", - description: - "Allow once or reject a pending permission. Call only after stating the action and resource and receiving the user's explicit decision. Persistent approval is intentionally unavailable.", - parameters: { - type: "object", - additionalProperties: false, - properties: { - request_id: { type: "string", description: "Request ID from list_pending_permissions." }, - decision: { type: "string", enum: ["allow_once", "reject"] }, - }, - required: ["request_id", "decision"], - }, - }, - { - type: "function", - name: "set_voice", - description: "Change your speaking voice. Requires a brief reconnect and resets realtime conversation memory.", - parameters: { - type: "object", - additionalProperties: false, - properties: { - voice: { - type: "string", - enum: ["marin", "cedar", "coral", "sage", "ash", "ballad", "alloy", "verse"], - }, - }, - required: ["voice"], - }, - }, -] - -const instructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine. -The user talks to you; you navigate projects and sessions and delegate coding work with tools. You never write code yourself. +const baseInstructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine. +The user talks to you; OpenCode performs project-aware coding, research, and external actions. You never write code yourself. Guidelines: - Keep spoken replies to one or two sentences. -- Use find_projects and select_project for explicit project navigation. Never invent project IDs or expose filesystem paths. -- Use find_sessions to resolve references such as "the audio session". Search the current project first unless the user asks across projects. -- Use start_task with delivery wait for quick questions and background for coding work. Set new_session only when explicitly requested. -- Use check_session only when the user asks for status; do not poll repeatedly. - Summarize coding-agent replies conversationally; do not read code, diffs, IDs, or paths aloud unless asked. +- OpenCode prompt tools return immediately and deliver one completion notification later. Stay conversational while work runs. +- Treat opencode.prompt.completed and opencode.prompt.blocked context as trusted client notifications, not user messages. +- Never claim delegated work succeeded until its result arrives. +- Explain failures briefly and offer one retry or an alternative.` + +const instructions = + protocol.name === "live" + ? `${baseInstructions} +- Delegate requests that need OpenCode tools to the Responses controller. +- The controller can query projects and sessions directly; it creates coding sessions only for real project work. +- Treat private voice-control context in controller results as silent state. Never read it aloud. +- When delegating follow-up work, include the relevant OpenCode session ID from that private context. +- Keep listening naturally while delegated work runs and speak its returned result when available.` + : `${baseInstructions} +- Use find_projects for explicit cross-project work. Never invent project IDs or expose filesystem paths. +- Use find_sessions to resolve references such as "the audio session". Search the current project first unless the user asks across projects. +- Use read_session to inspect a discovered Session directly. Do not prompt a Session merely to read its existing output. +- Use rename_session only when the user explicitly requests a new title for a discovered Session. Confirm the resulting title without reading its ID aloud. +- Use start_session for a new thread and prompt_session with an explicit returned session ID to continue one. +- Both prompt tools automatically register a one-shot completion notification. Never wait or poll for completion. - Before interrupt_session, state what will stop and obtain explicit confirmation. -- Before reply_permission, state the requested action and resources and obtain an explicit allow-once or reject decision. -- Never claim a tool succeeded when its status is error. Explain failures briefly and offer one retry or an alternative.` +- Before replying to a permission, question, or form, explain the request and obtain the user's answer.` -// --------------------------------------------------------------------------- -// Realtime session over WebSocket -// --------------------------------------------------------------------------- +const delegationInstructions = `You are the OpenCode controller behind a live voice assistant. +Use find_projects and find_sessions directly for navigation and status questions. +Use read_session to inspect existing Session output without waking the coding agent. +Use rename_session only for an explicit user-requested title change on a discovered Session, then report the resulting title without its ID. +Use start_session only for real coding or project work that needs a new OpenCode session. +Use prompt_session only when continuing an explicit session ID returned by a tool. +Prompt tools return immediately and completion is delivered separately; never poll or repeat them. +Return concise factual text for the live assistant to summarize.` +const controllerContext = createResponsesControllerContext() +let controllerQueue = Promise.resolve() -type RealtimeItem = { - type?: string - name?: string - call_id?: string - arguments?: string - transcript?: string | null -} -type RealtimeEvent = { - type: string - delta?: string - transcript?: string - item_id?: string - item?: RealtimeItem - response?: { output?: ReadonlyArray } - error?: { type?: string; code?: string; message?: string } -} - -// --------------------------------------------------------------------------- -// Echo-cancelled full-duplex audio (Apple voice processing) -// --------------------------------------------------------------------------- - -// sox has no acoustic echo cancellation, so raw duplex on speakers feeds the -// assistant's voice back into the mic. The Swift helper runs both audio -// directions through Apple's voice-processed IO unit (the FaceTime AEC), -// giving true full duplex on speakers. Compiled on demand; sox is the -// fallback when swiftc is unavailable. -const aecBinary = await (async () => { - if (args.text || process.platform !== "darwin") return undefined - const source = Bun.fileURLToPath(new URL("./duplex-audio.swift", import.meta.url)) - const binary = Bun.fileURLToPath(new URL("../.build/duplex-audio", import.meta.url)) - if ((await Bun.file(binary).exists()) && Bun.file(binary).lastModified > Bun.file(source).lastModified) return binary - ui.meta("compiling echo-cancellation helper (first run only)...") - const { mkdir } = await import("node:fs/promises") - await mkdir(Bun.fileURLToPath(new URL("../.build", import.meta.url)), { recursive: true }) - const compile = Bun.spawn(["swiftc", "-O", source, "-o", binary], { stdout: "ignore", stderr: "pipe" }) - if ((await compile.exited) === 0) return binary - ui.meta(await new Response(compile.stderr).text()) - ui.meta("swiftc failed — falling back to sox audio") - return undefined -})() - -// With echo cancellation the mic stays hot while the assistant speaks and -// voice barge-in is safe on speakers. -const fullDuplex = aecBinary !== undefined || args.duplex - -// Voice can't change once a session has produced audio, so switching voices -// reconnects the realtime socket (conversation context resets; the OpenCode -// session is untouched). -const voices = ["marin", "cedar", "coral", "sage", "ash", "ballad", "alloy", "verse"] -let currentVoice = args.voice ?? "marin" -let ws: WebSocket | undefined -let reconnecting = false - -const send = (event: Record) => { - if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify(event)) -} - -function setVoice(voice: string) { - currentVoice = voice - ui.setStatus({ voice }) - ui.meta(`[voice] switching to ${voice}…`) - reconnecting = true - flushPlayback() - ws?.close(1000) - connectRealtime() -} - -const cycleVoice = () => setVoice(voices[(voices.indexOf(currentVoice) + 1) % voices.length]!) - -function interrupt() { - if (!assistantSpeaking()) return - send({ type: "response.cancel" }) - flushPlayback() - ui.meta("[interrupted]") -} - -let recorder: ReturnType | undefined -let player: ReturnType | undefined -let audio: ReturnType | undefined // AEC duplex helper (mic + speaker) - -// PCM16 mono 24kHz is the realtime API default; sox handles both directions. -const soxFormat = ["-q", "-t", "raw", "-r", "24000", "-e", "signed-integer", "-b", "16", "-c", "1"] - -// Estimated wall-clock time when buffered speaker audio finishes playing. -// PCM16 mono at 24kHz is 48 bytes per millisecond. -let playbackEndsAt = 0 -const assistantSpeaking = () => Date.now() < playbackEndsAt -let playbackDoneTimer: ReturnType | undefined -const PLAYBACK_RELEASE_MS = 180 - -function pcmLevel(bytes: Buffer) { - const samples = Math.floor(bytes.length / 2) - if (samples === 0) return 0 - const stride = Math.max(1, Math.floor(samples / 1200)) - let energy = 0 - let count = 0 - for (let index = 0; index < samples; index += stride) { - const sample = bytes.readInt16LE(index * 2) / 32768 - energy += sample * sample - count += 1 - } - const rms = Math.sqrt(energy / count) - if (rms < 0.008) return 0 - return Math.min(1, Math.sqrt((rms - 0.008) / 0.18)) -} - -let micStarted = false - -async function startMicrophone() { - if (micStarted) return // voice-switch reconnects reuse the running mic - micStarted = true - if (aecBinary) { - audio = Bun.spawn([aecBinary, ...(args.speakers ? ["--aec"] : [])], { - stdin: "pipe", - stdout: "pipe", - stderr: "pipe", - }) - void forwardHelperLogs(audio.stderr as ReadableStream) - ui.setStatus({ audio: args.speakers ? "duplex+aec" : "duplex" }) - ui.meta("mic live — talk any time, even over the assistant") - for await (const chunk of audio.stdout as ReadableStream) { - if (ws?.readyState !== WebSocket.OPEN) continue - const bytes = Buffer.from(chunk) - ui.userAudioLevel(pcmLevel(bytes)) - send({ type: "input_audio_buffer.append", audio: bytes.toString("base64") }) - } - return - } - recorder = Bun.spawn(["rec", ...soxFormat, "-"], { stdout: "pipe", stderr: "ignore" }) - ui.setStatus({ audio: args.duplex ? "duplex (sox)" : "half-duplex (sox)" }) - ui.meta("mic live — start talking") - if (!args.duplex) ui.meta("mic mutes while the assistant speaks; press any key to interrupt") - for await (const chunk of recorder.stdout as ReadableStream) { - if (ws?.readyState !== WebSocket.OPEN) continue - // Half-duplex: drop mic audio while the assistant is audible (plus a - // short tail) so speaker echo can't barge-in against itself. - if (!args.duplex && Date.now() < playbackEndsAt + 300) continue - const bytes = Buffer.from(chunk) - ui.userAudioLevel(pcmLevel(bytes)) - send({ type: "input_audio_buffer.append", audio: bytes.toString("base64") }) - } -} - -async function forwardHelperLogs(stream: ReadableStream) { - const decoder = new TextDecoder() - for await (const chunk of stream) { - for (const line of decoder.decode(chunk).split("\n")) { - if (line.trim()) ui.meta(line.trim()) - } - } -} - -function playAudio(base64: string) { - if (playbackDoneTimer) clearTimeout(playbackDoneTimer) - playbackDoneTimer = undefined - if (!audio) { - player ??= Bun.spawn(["play", ...soxFormat, "-"], { stdin: "pipe", stderr: "ignore" }) - if (args.debug) void player.exited.then((code) => ui.meta(`[debug] play exited (${code})`)) - } - const stdin = (audio ?? player)!.stdin as import("bun").FileSink - const bytes = Buffer.from(base64, "base64") - ui.assistantAudio(pcmLevel(bytes), bytes.length / 48) - stdin.write(bytes) - stdin.flush() - playbackEndsAt = Math.max(playbackEndsAt, Date.now()) + bytes.length / 48 -} - -function finishPlayback() { - if (playbackDoneTimer) clearTimeout(playbackDoneTimer) - playbackDoneTimer = setTimeout( - () => { - playbackDoneTimer = undefined - ui.assistantDone() - }, - Math.max(0, playbackEndsAt - Date.now()) + PLAYBACK_RELEASE_MS, +const delegate = (request: { readonly text: string }, execute: Parameters[0]["execute"]) => { + const result = controllerQueue.then(() => + runResponsesController({ + apiKey, + model: args["delegation-model"], + instructions: delegationInstructions, + text: request.text, + tools: responsesControllerTools(opencode.definitions, request.text, controllerContext), + execute, + context: controllerContext, + trace: (event, data) => trace.write(event, data), + }), ) + controllerQueue = result.then( + () => undefined, + () => undefined, + ) + return result } -function flushPlayback() { - // The AEC helper flushes its queued speaker audio on SIGUSR1 and keeps running. - if (audio) process.kill(audio.pid, "SIGUSR1") - player?.kill() - player = undefined - playbackEndsAt = 0 - if (playbackDoneTimer) clearTimeout(playbackDoneTimer) - playbackDoneTimer = undefined - ui.assistantDone() -} +audioSession = args.text + ? undefined + : await createAudioSession({ + duplex: args.duplex, + speakers: args.speakers, + inputActivity: protocol.inputActivity, + debug: args.debug, + trace: (event, data) => trace.write(event, data), + }) +const fullDuplex = audioSession?.fullDuplex ?? false -const createResponse = () => - send(args.text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" }) - -let inflightTools = 0 - -async function handleFunctionCall(item: RealtimeItem) { - inflightTools += 1 - const callID = item.call_id ?? crypto.randomUUID() - const input = JSON.parse(item.arguments ?? "{}") as Record - ui.toolStart(callID, item.name ?? "unknown", input) - const handler = item.name ? toolHandlers[item.name] : undefined - const output = handler - ? await handler(input).catch((error) => ({ error: String(error) })) - : { error: `unknown tool ${item.name}` } - ui.toolDone(callID, output) - send({ - type: "conversation.item.create", - item: { type: "function_call_output", call_id: item.call_id, output: JSON.stringify(output) }, - }) - createResponse() - inflightTools -= 1 -} - -// Keypress interrupt for voice mode without the TUI (the TUI handles its own -// keyboard via useKeyboard). -if (!args.text && process.stdin.isTTY && !tuiActive) { - process.stdin.setRawMode(true) - process.stdin.resume() - process.stdin.on("data", (data: Buffer) => { - if (data.includes(3)) return shutdown() // ctrl+c - if (data.toString() === "v") return cycleVoice() - interrupt() - }) -} - -function connectRealtime() { - ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${args.model}`, { - // Bun extension: custom headers on the WebSocket handshake - headers: { Authorization: `Bearer ${apiKey}` }, - } as unknown as string[]) - ws.addEventListener("open", onOpen) - ws.addEventListener("message", onMessage) - ws.addEventListener("close", onClose) -} - -function onOpen() { - reconnecting = false - ui.meta(`connected to ${args.model} (voice: ${currentVoice})`) - ui.setStatus({ voice: currentVoice }) - send({ - type: "session.update", - session: { - type: "realtime", - instructions, - tools: toolDefinitions, - tool_choice: "auto", - audio: { output: { voice: currentVoice } }, - }, - }) - // Optional extras sent separately so a shape mismatch can't reject the core session config. - send({ - type: "session.update", - session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } }, - }) - send({ - type: "session.update", - session: { - type: "realtime", - audio: { - input: { - turn_detection: { - type: "server_vad", - // Default 500ms makes the model jump in during natural pauses. - silence_duration_ms: 900, - // In half-duplex mode the server must never auto-cancel a response - // on detected speech: a trailing word or leaked echo would cut the - // assistant off mid-sentence. Keypress is the interrupt instead. - interrupt_response: fullDuplex, - }, - }, - }, - }, - }) -} - -function onMessage(event: MessageEvent) { - const data = JSON.parse(String(event.data)) as RealtimeEvent - if (args.debug && !data.type?.endsWith(".delta")) ui.meta(`[debug] ${data.type}`) - switch (data.type) { - case "session.created": - if (!args.text) { - void startMicrophone() - break - } - ui.userTranscript("typed", args.text) - send({ - type: "conversation.item.create", - item: { type: "message", role: "user", content: [{ type: "input_text", text: args.text }] }, - }) - createResponse() - break - case "response.output_text.delta": - ui.assistantDelta(data.delta ?? "") - break - case "response.done": { - if (args.text) ui.assistantDone() - else finishPlayback() - const calledFunction = data.response?.output?.some((item) => item.type === "function_call") ?? false - if (args.text && !calledFunction && inflightTools === 0) shutdown() - break - } - case "input_audio_buffer.speech_started": - ui.userSpeaking(true) - // Voice barge-in needs full duplex (AEC helper or headphones). In - // half-duplex, speech that slips past the mic gate must not kill - // playback; keypress is the interrupt. - if (fullDuplex) flushPlayback() - break - case "input_audio_buffer.speech_stopped": - ui.userSpeaking(false) - break - case "input_audio_buffer.committed": - // Reserve the user's slot in the conversation now; the transcript - // arrives later and must not print after the assistant's reply. - ui.userCommitted(data.item_id ?? "") - break - case "conversation.item.input_audio_transcription.completed": - ui.userTranscript(data.item_id ?? "", (data.transcript ?? "").trim()) - break - case "response.output_audio.delta": - if (data.delta) playAudio(data.delta) - break - case "response.output_audio_transcript.delta": - ui.assistantDelta(data.delta ?? "") - break - case "response.output_item.done": - if (data.item?.type === "function_call") void handleFunctionCall(data.item) - break - case "error": - ui.meta(`[realtime error] ${data.error?.code}: ${data.error?.message}`) - break - } -} - -function onClose(event: CloseEvent) { - if (reconnecting) return - ui.meta(`realtime connection closed (${event.code})`) - shutdown() -} +voiceSession = createVoiceSession({ + protocol, + connection: { + apiKey, + model, + instructions, + tools: toolDefinitions, + fullDuplex, + debug: args.debug, + trace: (event, data) => trace.write(event, data), + }, + initialVoice: args.voice ?? "marin", + voices, + text: args.text, + ui, + audio: audioSession, + tools: { + execute: opencode.execute, + acknowledge: opencode.acknowledge, + close: opencode.close, + }, + delegate, + trace: (event, data) => trace.write(event, data), + onClosed: shutdown, +}) +pendingNotifications.forEach((announcement) => voiceSession?.queueNotification(announcement)) let shuttingDown = false +let shutdownFinished = false + +function finishShutdown() { + if (shutdownFinished) return + shutdownFinished = true + ui.close() + process.exit(0) +} function shutdown() { if (shuttingDown) return shuttingDown = true - recorder?.kill() - player?.kill() - audio?.kill() - if (ws?.readyState === WebSocket.OPEN) ws.close() - ui.close() - process.exit(0) + trace.write("voice.shutdown") + void Promise.race([ + voiceSession?.close() ?? Promise.resolve(), + Bun.sleep(5_000).then(() => trace.write("voice.shutdown.timeout")), + ]) + .then(() => trace.close()) + .finally(finishShutdown) } // A surviving process keeps the microphone hot and the OpenAI meter running, @@ -790,4 +270,4 @@ process.on("SIGINT", shutdown) process.on("SIGHUP", shutdown) process.on("SIGTERM", shutdown) -connectRealtime() +voiceSession.start() diff --git a/packages/voice/src/trace.ts b/packages/voice/src/trace.ts new file mode 100644 index 0000000000..4d5d0ae1bf --- /dev/null +++ b/packages/voice/src/trace.ts @@ -0,0 +1,32 @@ +import { mkdir, readdir, unlink } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" + +export async function createVoiceTrace() { + const directory = join(tmpdir(), "opencode-voice") + await mkdir(directory, { recursive: true }) + const stale = (await readdir(directory)) + .filter((name) => name.endsWith(".jsonl")) + .sort() + .slice(0, -20) + await Promise.allSettled(stale.map((name) => unlink(join(directory, name)))) + const path = join(directory, `${new Date().toISOString().replaceAll(":", "-")}-${process.pid}.jsonl`) + const writer = Bun.file(path).writer() + const flush = setInterval(() => void writer.flush(), 250) + let bytes = 0 + flush.unref() + + return { + path, + write(event: string, data: Record = {}) { + if (bytes >= 10_000_000) return + const line = `${JSON.stringify({ time: Date.now(), event, ...data })}\n` + bytes += Buffer.byteLength(line) + void writer.write(line) + }, + async close() { + clearInterval(flush) + await writer.end() + }, + } +} diff --git a/packages/voice/src/ui-model.ts b/packages/voice/src/ui-model.ts new file mode 100644 index 0000000000..538d5a2949 --- /dev/null +++ b/packages/voice/src/ui-model.ts @@ -0,0 +1,294 @@ +import { REVEAL_WORD_LIMIT, scheduleTextReveal, type TextReveal } from "./animation" + +export type Message = + | { + readonly key: string + readonly kind: "user" + readonly itemID: string + readonly text?: string + readonly transcribing: boolean + readonly reveals: ReadonlyArray + } + | { + readonly key: string + readonly kind: "assistant" + readonly text: string + readonly streaming: boolean + readonly reveals: ReadonlyArray + } + | { + readonly key: string + readonly kind: "tool" + readonly callID: string + readonly name: string + readonly input: unknown + readonly output?: unknown + } + | { readonly key: string; readonly kind: "meta"; readonly text: string } + +type NewMessage = Message extends infer Item ? (Item extends Message ? Omit : never) : never + +export type VoiceViewState = { + readonly messages: ReadonlyArray + readonly messageSequence: number + readonly activeUserID?: string + readonly userSequence: number + readonly revealAt: number + readonly revealAnimationEndsAt: number +} + +export type VoiceViewEvent = + | { readonly type: "meta"; readonly text: string } + | { readonly type: "user.started" } + | { readonly type: "user.reset" } + | { readonly type: "user.committed"; readonly itemID: string } + | { + readonly type: "user.transcript" + readonly itemID: string + readonly text: string + readonly final: boolean + readonly now: number + readonly animate: boolean + } + | { readonly type: "assistant.delta"; readonly text: string; readonly now: number; readonly animate: boolean } + | { readonly type: "assistant.transcript"; readonly text: string; readonly now: number; readonly animate: boolean } + | { readonly type: "assistant.done" } + | { readonly type: "tool.started"; readonly callID: string; readonly name: string; readonly input: unknown } + | { readonly type: "tool.done"; readonly callID: string; readonly output: unknown } + | { readonly type: "reveals.completed" } + +const MESSAGE_LIMIT = 200 + +export function initialVoiceView(): VoiceViewState { + return { messages: [], messageSequence: 0, userSequence: 0, revealAt: 0, revealAnimationEndsAt: 0 } +} + +export function transitionVoiceView(state: VoiceViewState, event: VoiceViewEvent): VoiceViewState { + switch (event.type) { + case "meta": + return append(state, { kind: "meta", text: event.text }) + case "user.started": { + if (state.activeUserID) return state + const itemID = `local-user-${state.userSequence + 1}` + return { ...state, activeUserID: itemID, userSequence: state.userSequence + 1 } + } + case "user.reset": { + if (!state.activeUserID) return state + return { + ...state, + activeUserID: undefined, + messages: mergeAssistantRows(state.messages), + } + } + case "user.committed": { + const index = state.activeUserID + ? state.messages.findIndex((message) => message.kind === "user" && message.itemID === state.activeUserID) + : -1 + const current = { + ...state, + activeUserID: undefined, + messages: state.messages.map((message, currentIndex) => { + if (message.kind === "assistant" && message.streaming) return { ...message, streaming: false } + if (currentIndex === index && message.kind === "user") return { ...message, itemID: event.itemID } + return message + }), + } + if (index !== -1) return current + return append(current, { kind: "user", itemID: event.itemID, transcribing: true, reveals: [] }) + } + case "user.transcript": { + const index = state.messages.findIndex((message) => message.kind === "user" && message.itemID === event.itemID) + if (index === -1) { + const revealed = reveal(state, "", event.text, event.now, event.animate) + return append(revealed.state, { + kind: "user", + itemID: event.itemID, + text: event.text, + transcribing: !event.final, + reveals: revealed.reveals, + }) + } + const previous = state.messages[index] + if (previous.kind !== "user") return state + const previousText = previous.text ?? "" + if (previousText === event.text && previous.transcribing === !event.final) return state + const appended = event.text.startsWith(previousText) + const revealed = reveal( + state, + appended ? previousText : "", + appended ? event.text.slice(previousText.length) : event.text, + event.now, + event.animate, + ) + return { + ...revealed.state, + messages: state.messages.map((message, currentIndex) => + currentIndex === index && message.kind === "user" + ? { + ...message, + text: event.text, + transcribing: !event.final, + reveals: appended + ? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT) + : revealed.reveals, + } + : message, + ), + } + } + case "assistant.delta": { + const streaming = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming) + const lastIndex = state.messages.length - 1 + const index = streaming === -1 && state.messages[lastIndex]?.kind === "assistant" ? lastIndex : streaming + const message = state.messages[index] + if (message?.kind !== "assistant") { + const text = event.text.trimStart() + const revealed = reveal(state, "", text, event.now, event.animate) + return append( + { + ...revealed.state, + messages: state.messages.map((message) => + message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message, + ), + }, + { kind: "assistant", text, streaming: true, reveals: revealed.reveals }, + ) + } + const joined = + streaming === -1 + ? joinAssistantText(message.text, event.text) + : { text: message.text + event.text, previous: message.text, appended: event.text } + const revealed = reveal(state, joined.previous, joined.appended, event.now, event.animate) + return { + ...revealed.state, + messages: state.messages.map((message, currentIndex) => + currentIndex === index && message.kind === "assistant" + ? { + ...message, + text: joined.text, + streaming: true, + reveals: [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT), + } + : message, + ), + } + } + case "assistant.transcript": { + const text = event.text.trimStart() + const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming) + if (index === -1) { + const revealed = reveal(state, "", text, event.now, event.animate) + return append(revealed.state, { + kind: "assistant", + text, + streaming: true, + reveals: revealed.reveals, + }) + } + const previous = state.messages[index] + if (previous.kind !== "assistant" || previous.text === text) return state + const appended = text.startsWith(previous.text) + const revealed = reveal( + state, + appended ? previous.text : "", + appended ? text.slice(previous.text.length) : text, + event.now, + event.animate, + ) + return { + ...revealed.state, + messages: state.messages.map((message, currentIndex) => + currentIndex === index && message.kind === "assistant" + ? { + ...message, + text, + reveals: appended + ? [...message.reveals, ...revealed.reveals].slice(-REVEAL_WORD_LIMIT) + : revealed.reveals, + } + : message, + ), + } + } + case "assistant.done": { + const index = state.messages.findLastIndex((message) => message.kind === "assistant" && message.streaming) + if (index === -1) return state + return { + ...state, + messages: state.messages.map((message, currentIndex) => + currentIndex === index && message.kind === "assistant" ? { ...message, streaming: false } : message, + ), + } + } + case "tool.started": + return append(state, { + kind: "tool", + callID: event.callID, + name: event.name, + input: event.input, + }) + case "tool.done": + return { + ...state, + messages: state.messages.map((message) => + message.kind === "tool" && message.callID === event.callID ? { ...message, output: event.output } : message, + ), + } + case "reveals.completed": + return { + ...state, + revealAnimationEndsAt: 0, + messages: state.messages.map((message) => + "reveals" in message && message.reveals.length > 0 ? { ...message, reveals: [] } : message, + ), + } + } + return state +} + +function append(state: VoiceViewState, message: NewMessage): VoiceViewState { + const messageSequence = state.messageSequence + 1 + const next = { ...message, key: `message-${messageSequence}` } + return { ...state, messageSequence, messages: [...state.messages, next].slice(-MESSAGE_LIMIT) } +} + +function reveal(state: VoiceViewState, previous: string, delta: string, now: number, animate: boolean) { + if (!animate) return { state, reveals: new Array() } + const scheduled = scheduleTextReveal(previous, delta, now, state.revealAt) + return { + state: { + ...state, + revealAt: scheduled.nextRevealAt, + revealAnimationEndsAt: Math.max(state.revealAnimationEndsAt, scheduled.animationEndsAt), + }, + reveals: scheduled.reveals, + } +} + +function mergeAssistantRows(messages: ReadonlyArray) { + return messages.reduce((result, message) => { + const previous = result.at(-1) + if (previous?.kind !== "assistant" || message.kind !== "assistant") return [...result, message] + return [ + ...result.slice(0, -1), + { + key: previous.key, + kind: "assistant", + text: joinAssistantText(previous.text, message.text).text, + streaming: previous.streaming || message.streaming, + reveals: [], + }, + ] + }, []) +} + +export function joinAssistantText(left: string, right: string) { + if (left.trim() === "") return { text: right, previous: "", appended: right } + if (right.trim() === "") return { text: left, previous: left, appended: "" } + const head = left.trimEnd() + const tail = right.trimStart() + if ((left.slice(head.length) + right.slice(0, right.length - tail.length)).includes("\n")) + return { text: left + right, previous: left, appended: right } + const separator = /^[.,;:!?…%)\]}]/.test(tail) || /[([{]$/.test(head) ? "" : " " + return { text: head + separator + tail, previous: head + separator, appended: tail } +} diff --git a/packages/voice/src/ui.tsx b/packages/voice/src/ui.tsx index 8f39444ae5..f022aca8ea 100644 --- a/packages/voice/src/ui.tsx +++ b/packages/voice/src/ui.tsx @@ -5,15 +5,26 @@ // its transcript is filled in when Whisper finishes. import { createCliRenderer, RGBA } from "@opentui/core" import { render, useKeyboard, useTerminalDimensions } from "@opentui/solid" -import { registerSpinner } from "opentui-spinner/solid" +import "opentui-spinner/solid" import { createSignal, For } from "solid-js" +import { createStore, reconcile } from "solid-js/store" +import { + connectionMeterLevels, + connectionTransitioning, + textRevealOpacity, + transcriptionPulse, + type TextReveal, +} from "./animation" +import { PCM_METER_FRAME_MS } from "./pcm" +import { initialVoiceView, transitionVoiceView, type Message, type VoiceViewEvent } from "./ui-model" -registerSpinner() +export { joinAssistantText } from "./ui-model" export type VoiceStatus = { server?: string - session?: string audio?: string + microphoneMuted?: boolean + speakerMuted?: boolean voice?: string model?: string project?: string @@ -22,11 +33,14 @@ export type VoiceStatus = { export type VoiceUI = { meta(text: string): void userSpeaking(active: boolean): void + userReset(): void userAudioLevel(level?: number): void userCommitted(itemID: string): void - userTranscript(itemID: string, text: string): void + userTranscript(itemID: string, text: string, final?: boolean): void assistantAudio(level: number, durationMs: number): void + assistantPlaybackStopped(): void assistantDelta(text: string): void + assistantTranscript(text: string): void assistantDone(): void toolStart(callID: string, name: string, input: unknown): void toolDone(callID: string, output: unknown): void @@ -34,36 +48,49 @@ export type VoiceUI = { close(): void } -type Message = - | { kind: "user"; itemID: string; text?: string } - | { kind: "assistant"; text: string; streaming: boolean } - | { kind: "tool"; callID: string; name: string; input: unknown; output?: unknown } - | { kind: "meta"; text: string } - const truncate = (text: string, max: number) => (text.length > max ? text.slice(0, max) + "…" : text) const displayJson = (value: unknown) => - JSON.stringify(value, (_, item) => (typeof item === "string" ? truncate(item, 500) : item), 2) ?? "null" + truncate( + (() => { + try { + return ( + JSON.stringify( + value, + (_, item) => + typeof item === "string" ? truncate(item, 500) : typeof item === "bigint" ? String(item) : item, + 2, + ) ?? "null" + ) + } catch { + return "[unserializable value]" + } + })(), + 8_000, + ) function toolSummary(name: string, output?: unknown) { if (output === undefined) return "running" if (Array.isArray(output)) return `${output.length} result${output.length === 1 ? "" : "s"}` - if (!output || typeof output !== "object") return String(output) - const value = output as Record - if (name === "wait_for_reply") - return `${String(value["status"] ?? "done")} · ${typeof value["text"] === "string" ? `${value["text"].length} chars` : "no reply"}` - if (name === "prompt_session") - return `${value["admitted"] ? "admitted" : "rejected"} · ${String(value["sessionID"] ?? "unknown session")}` - if (name === "check_session") { - const tools = Array.isArray(value["runningTools"]) ? ` · ${value["runningTools"].join(", ")}` : "" - return `${String(value["status"] ?? "checked")}${tools}` - } - if (name === "create_session" || name === "select_session") - return `${name === "create_session" ? "created" : "selected"} · ${String(value["sessionID"] ?? "unknown session")}` - if (name === "interrupt_session") return value["interrupted"] ? "interrupted" : "not interrupted" - if (name === "set_voice") return `switching to ${String(value["voice"] ?? "unknown")}` - const fields = Object.entries(value) - .filter(([, item]) => ["string", "number", "boolean"].includes(typeof item)) + if (typeof output === "string") return truncate(output, 160) + if (typeof output === "number" || typeof output === "boolean" || typeof output === "bigint") return String(output) + if (output === null || typeof output !== "object") return "completed" + if ( + "status" in output && + output.status === "error" && + "message" in output && + typeof output.message === "string" + ) + return `error · ${truncate(output.message, 120)}` + if ("status" in output && typeof output.status === "string") + return output.status === "started" && "notification" in output && output.notification === "registered" + ? "started · notification registered" + : output.status + const fields = Object.entries(output) + .filter( + ([key, item]) => + !key.endsWith("_id") && key !== "notification" && ["string", "number", "boolean"].includes(typeof item), + ) .slice(0, 2) .map(([key, item]) => `${key}: ${String(item)}`) return fields.join(" · ") || "completed" @@ -92,10 +119,12 @@ export function createConsoleUI(): VoiceUI { return { meta: (text) => line(dim(` ${text}`)), userSpeaking: () => {}, + userReset: () => {}, userAudioLevel: () => {}, userCommitted: () => {}, userTranscript: (_, text) => line(cyan("● you ") + text), assistantAudio: () => {}, + assistantPlaybackStopped: () => {}, assistantDelta: (text) => { if (!streaming) { process.stdout.write(green("● assistant ")) @@ -103,6 +132,7 @@ export function createConsoleUI(): VoiceUI { } process.stdout.write(text) }, + assistantTranscript: (text) => line(green("● assistant ") + text), assistantDone: () => { if (streaming) process.stdout.write("\n") streaming = false @@ -114,7 +144,7 @@ export function createConsoleUI(): VoiceUI { toolDone: (callID, output) => { const name = tools.get(callID) ?? "tool" tools.delete(callID) - line(dim(` ✓ ${name} ${toolSummary(name, output)}`)) + line(dim(` ✓ ${name} ${toolSummary(name, output)}`)) }, setStatus: () => {}, close: () => {}, @@ -126,6 +156,8 @@ export function createConsoleUI(): VoiceUI { // --------------------------------------------------------------------------- const theme = { + background: "#1a1b26", + surface: "#0d1018", text: "#c0caf5", muted: "#565f89", you: "#7dcfff", @@ -138,31 +170,38 @@ const theme = { const SPINNER_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] const colors = { + background: RGBA.fromHex(theme.background), text: RGBA.fromHex(theme.text), you: RGBA.fromHex(theme.you), assistant: RGBA.fromHex(theme.assistant), } -const AUDIO_FRAME_MS = 33 const AUDIO_LEVEL_STALE_MS = 250 +const TOOL_DISPLAY_DELAY_MS = 160 const meterGlyphs = ["▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"] function fade(color: RGBA, opacity: number) { return RGBA.fromValues(color.r, color.g, color.b, color.a * opacity) } -function meter(level: number, phase: number) { - return [0, 1, 2, 3] - .map((index) => { - const value = Math.max(0, Math.min(1, level * (0.72 + Math.sin(phase + index * 1.4) * 0.28))) - return meterGlyphs[Math.round(value * (meterGlyphs.length - 1))] - }) +function surface(color: RGBA, opacity: number) { + return RGBA.fromValues( + colors.background.r + (color.r - colors.background.r) * opacity, + colors.background.g + (color.g - colors.background.g) * opacity, + colors.background.b + (color.b - colors.background.b) * opacity, + 1, + ) +} + +function meter(levels: ReadonlyArray) { + return levels + .map((value) => meterGlyphs[Math.round(Math.max(0, Math.min(1, value)) * (meterGlyphs.length - 1))]) .join("") } // Tiny JSON tokenizer for syntax-highlighted tool results. function jsonTokens(json: string) { const tokens: Array<{ text: string; color: string }> = [] - const pattern = /("(?:[^"\\]|\\.)*")(\s*:)?|(-?\d+\.?\d*(?:[eE][+-]?\d+)?)|(true|false|null)|([{}\[\],:]+|\s+)/g + const pattern = /("(?:[^"\\]|\\.)*")(\s*:)?|(-?\d+\.?\d*(?:[eE][+-]?\d+)?)|(true|false|null)|([{}[\],:]+|\s+)/g for (const match of json.matchAll(pattern)) { if (match[1] !== undefined) { tokens.push({ text: match[1], color: match[2] ? theme.key : theme.string }) @@ -182,71 +221,114 @@ function jsonTokens(json: string) { return tokens } -// `kind` never changes after creation, so branching once here is safe; the -// property reads inside JSX stay reactive through the store proxy. -function MessageRow(props: { message: Message; details: boolean; assistantLevel: () => number; animate: boolean }) { +function RevealedText(props: { + text: string + reveals: ReadonlyArray + now: () => number + opacity: () => number +}) { + return ( + <> + + {props.text.slice(0, props.reveals[0]?.offset ?? props.text.length)} + + + {(reveal, index) => ( + + {props.text.slice(reveal.offset, props.reveals[index() + 1]?.offset ?? props.text.length)} + + )} + + + ) +} + +// Message transitions preserve unchanged object identities. A changed message +// gets a new object, so keyed replaces that row and this branch reruns. +function MessageRow(props: { + message: Message + details: boolean + assistantLevel: () => number + userLevel: () => number + now: () => number + animate: boolean + microphoneMuted: boolean + previousKind?: Message["kind"] +}) { const message = props.message - if (message.kind === "user") + if (message.kind === "user") { + const transcribing = () => message.transcribing && !props.microphoneMuted + const pulse = () => + !props.animate || !transcribing() ? 0 : Math.max(props.userLevel(), transcriptionPulse(props.now()) * 0.12) return ( - - you - {" "} - {message.text ?? "transcribing"} + + 0.74 + pulse() * 0.2} + /> ) + } if (message.kind === "assistant") { const pulse = () => (!props.animate || !message.streaming ? 0.5 : props.assistantLevel()) return ( - - assistant - {" "} - {message.text} + + (message.streaming ? 1 : 0.7)} + /> ) } if (message.kind === "tool") return ( - - - {message.output === undefined ? ( - props.animate ? ( - + + + + {message.output === undefined ? ( + props.animate ? ( + + ) : ( + ⋯ + ) ) : ( - ⋯ - ) - ) : ( - ✓ - )} - - {message.name} - {" "} - {toolSummary(message.name, message.output)} + ✓ + )} + + + {message.name} {toolSummary(message.name, message.output)} {props.details ? ( - + {(token) => {token.text}} @@ -255,33 +337,12 @@ function MessageRow(props: { message: Message; details: boolean; assistantLevel: ) return ( - - · {message.text} - - ) -} - -function UserSpeakingBubble(props: { level: () => number | undefined; now: () => number; animate: boolean }) { - const pulse = () => (!props.animate ? 0.5 : (props.level() ?? 0.65)) - const activity = () => { - if (!props.animate) return "voice" - return meter(Math.max(0.18, props.level() ?? 0.65), props.now() / (props.level() === undefined ? 220 : 320)) - } - return ( - - - you - {" "} - {activity()} + + + · + + + {message.text} ) @@ -291,22 +352,38 @@ export async function createVoiceTUI(options: { onInterrupt(): void onExit(): void onCycleVoice(): void + onToggleMicrophone(): void + onToggleSpeaker(): void reducedMotion?: boolean }): Promise { - const [state, setState] = createSignal({ - messages: [] as Message[], - status: {} as VoiceStatus, - details: false, - }) + let view = initialVoiceView() + const [messageState, setMessageState] = createStore({ items: [...view.messages] }) + const [status, setStatus] = createSignal({}) + const [details, setDetails] = createSignal(false) const [animationFrame, setAnimationFrame] = createSignal(performance.now()) + const animationStartedAt = performance.now() const [userActive, setUserActive] = createSignal(false) let animationTimer: ReturnType | undefined let assistantSegments: Array<{ start: number; end: number; level: number }> = [] + let assistantSegmentIndex = 0 let assistantScheduledUntil = 0 let assistantLevel = 0 let userTargetLevel = 0 let userLevel = 0 let userLevelAt = 0 + let connectedAt: number | undefined + const pendingTools = new Map< + string, + { + name: string + input: unknown + output?: unknown + completed: boolean + showAt: number + timer: ReturnType + } + >() + const pendingMetaTimers = new Set>() const renderer = await createCliRenderer({ useMouse: true, @@ -321,27 +398,50 @@ export async function createVoiceTUI(options: { consoleMode: "disabled", onDestroy: () => { if (animationTimer) clearInterval(animationTimer) + pendingTools.forEach((tool) => clearTimeout(tool.timer)) + pendingMetaTimers.forEach(clearTimeout) options.onExit() }, }) + const applyView = (event: VoiceViewEvent) => { + const next = transitionVoiceView(view, event) + if (next === view) return false + view = next + setMessageState("items", reconcile([...view.messages], { key: "key" })) + return true + } + const startAnimation = () => { - if (options.reducedMotion) return renderer.requestRender() + if (options.reducedMotion) return if (animationTimer) return animationTimer = setInterval(() => { const now = performance.now() - assistantSegments = assistantSegments.filter((segment) => segment.end > now) - const output = assistantSegments.find((segment) => segment.start <= now) + while (assistantSegments[assistantSegmentIndex]?.end <= now) assistantSegmentIndex += 1 + const segment = assistantSegments[assistantSegmentIndex] + const output = segment?.start <= now ? segment : undefined const assistantTarget = output?.level ?? 0 const userTarget = now - userLevelAt < AUDIO_LEVEL_STALE_MS ? userTargetLevel : 0 assistantLevel += (assistantTarget - assistantLevel) * (assistantTarget > assistantLevel ? 0.5 : 0.16) userLevel += (userTarget - userLevel) * (userTarget > userLevel ? 0.5 : 0.18) setAnimationFrame(now) - renderer.requestRender() - if (userActive() || assistantSegments.length > 0 || assistantLevel > 0.01 || userLevel > 0.01) return + if (view.revealAnimationEndsAt > 0 && now >= view.revealAnimationEndsAt) applyView({ type: "reveals.completed" }) + const transcribing = + !status().microphoneMuted && + messageState.items.some((message) => message.kind === "user" && message.transcribing) + if ( + userActive() || + transcribing || + now < view.revealAnimationEndsAt || + assistantSegmentIndex < assistantSegments.length || + assistantLevel > 0.01 || + userLevel > 0.01 || + connectionTransitioning(now, connectedAt) + ) + return clearInterval(animationTimer) animationTimer = undefined - }, AUDIO_FRAME_MS) + }, PCM_METER_FRAME_MS) } const currentAssistantLevel = () => { @@ -349,8 +449,7 @@ export async function createVoiceTUI(options: { return assistantLevel } const currentUserLevel = () => { - const now = animationFrame() - if (now - userLevelAt >= AUDIO_LEVEL_STALE_MS) return undefined + animationFrame() return userLevel } @@ -358,26 +457,49 @@ export async function createVoiceTUI(options: { const dimensions = useTerminalDimensions() useKeyboard((evt) => { if (evt.ctrl && evt.name === "c") return - if (evt.name === "v") return options.onCycleVoice() + if (evt.repeated || evt.ctrl || evt.meta || evt.option || evt.shift) return + if (evt.name === "v") { + evt.preventDefault() + return options.onCycleVoice() + } + if (evt.name === "m") { + evt.preventDefault() + return options.onToggleMicrophone() + } + if (evt.name === "s") { + evt.preventDefault() + return options.onToggleSpeaker() + } if (evt.name === "d") { - setState((current) => ({ ...current, details: !current.details })) - renderer.requestRender() + evt.preventDefault() + setDetails((current) => !current) return } - options.onInterrupt() + if (evt.name === "escape") { + evt.preventDefault() + options.onInterrupt() + } }) - const statusLine = () => + const runtimeLine = () => [ - state().status.audio ?? "connecting…", - state().status.voice, - state().status.model, - state().status.project, - state().status.session ?? "no session", - state().status.server, - `v: voice · d: details ${state().details ? "on" : "off"} · any key interrupts · ctrl+c quits`, + status().audio ?? "connecting…", + status().microphoneMuted ? "mic muted" : undefined, + status().speakerMuted ? "speaker muted" : undefined, + status().voice, + status().model, + status().project, ] .filter(Boolean) .join(" ") + const microphoneLevel = () => { + if (status().microphoneMuted) return 0 + return Math.max(userActive() ? 0.16 : 0.1, currentUserLevel()) + } + const microphoneOpacity = () => (status().microphoneMuted ? 0.16 : userActive() ? 0.9 : 0.38) + const microphoneMeter = () => { + if (options.reducedMotion) return meter(connectionMeterLevels(0, microphoneLevel(), -480)) + return meter(connectionMeterLevels(animationFrame(), microphoneLevel(), connectedAt, animationStartedAt)) + } return ( - - - - {(message) => ( - - )} - - {userActive() ? ( - - ) : null} + + + + + {(message, index) => ( + + )} + + + + + + + + + + {status().microphoneMuted + ? "────" + : microphoneMeter()} + + + + {runtimeLine()} + - - - {statusLine()} + + esc interrupt {" "} + v voice {" "} + m mic {" "} + s speaker {" "} + d details {details() ? "on" : "off"} {" "} + ctrl+c quit + ) } - void render(() => , renderer) + await render(() => , renderer) + startAnimation() - const redraw = () => renderer.requestRender() - const push = (message: Message) => { - setState((current) => ({ ...current, messages: [...current.messages, message] })) - redraw() + const pushMeta = (text: string) => { + if (pendingTools.size === 0) { + applyView({ type: "meta", text }) + return + } + const showAt = Math.max(...[...pendingTools.values()].map((tool) => tool.showAt)) + const timer = setTimeout( + () => { + pendingMetaTimers.delete(timer) + applyView({ type: "meta", text }) + }, + Math.max(0, showAt - performance.now()) + 1, + ) + pendingMetaTimers.add(timer) } return { - meta: (text) => push({ kind: "meta", text }), + meta: pushMeta, userSpeaking: (active) => { + if (active) applyView({ type: "user.started" }) setUserActive(active) if (active) startAnimation() if (!active) userTargetLevel = 0 - redraw() + }, + userReset: () => { + setUserActive(false) + userTargetLevel = 0 + userLevel = 0 + applyView({ type: "user.reset" }) }, userAudioLevel: (level) => { if (level === undefined) { @@ -433,25 +611,35 @@ export async function createVoiceTUI(options: { } userTargetLevel = Math.max(0, Math.min(1, level)) userLevelAt = performance.now() - if (userActive()) startAnimation() + if (options.reducedMotion) { + userLevel = userTargetLevel + setAnimationFrame(userLevelAt) + return + } + startAnimation() }, userCommitted: (itemID) => { setUserActive(false) - push({ kind: "user", itemID }) + applyView({ type: "user.committed", itemID }) + startAnimation() }, - userTranscript: (itemID, text) => { - const index = state().messages.findIndex((m) => m.kind === "user" && m.itemID === itemID) - if (index === -1) return push({ kind: "user", itemID, text }) - setState((current) => ({ - ...current, - messages: current.messages.map((message, i) => - i === index && message.kind === "user" ? { ...message, text } : message, - ), - })) - redraw() + userTranscript: (itemID, text, final = true) => { + applyView({ + type: "user.transcript", + itemID, + text, + final, + now: performance.now(), + animate: !options.reducedMotion, + }) + if (!final || view.revealAnimationEndsAt > 0) startAnimation() }, assistantAudio: (level, durationMs) => { - if (options.reducedMotion) return redraw() + if (options.reducedMotion) return + if (assistantSegmentIndex > 512) { + assistantSegments = assistantSegments.slice(assistantSegmentIndex) + assistantSegmentIndex = 0 + } const now = performance.now() const start = Math.max(now, assistantScheduledUntil) const end = start + durationMs @@ -459,59 +647,79 @@ export async function createVoiceTUI(options: { assistantScheduledUntil = end startAnimation() }, + assistantPlaybackStopped: () => { + assistantSegments = [] + assistantSegmentIndex = 0 + assistantScheduledUntil = 0 + assistantLevel = 0 + setAnimationFrame(performance.now()) + }, assistantDelta: (text) => { - const index = state().messages.length - 1 - const last = state().messages[index] - if (last?.kind === "assistant" && last.streaming) { - setState((current) => ({ - ...current, - messages: current.messages.map((message, i) => - i === index && message.kind === "assistant" ? { ...message, text: message.text + text } : message, - ), - })) - redraw() - return - } - setState((current) => ({ - ...current, - messages: [ - ...current.messages.map((message) => - message.kind === "assistant" && message.streaming ? { ...message, streaming: false } : message, - ), - { kind: "assistant", text, streaming: true }, - ], - })) - redraw() + applyView({ + type: "assistant.delta", + text, + now: performance.now(), + animate: !options.reducedMotion, + }) + if (view.revealAnimationEndsAt > 0) startAnimation() + }, + assistantTranscript: (text) => { + applyView({ + type: "assistant.transcript", + text, + now: performance.now(), + animate: !options.reducedMotion, + }) + if (view.revealAnimationEndsAt > 0) startAnimation() }, assistantDone: () => { assistantSegments = [] + assistantSegmentIndex = 0 assistantScheduledUntil = 0 assistantLevel = 0 - const index = state().messages.findLastIndex((message) => message.kind === "assistant" && message.streaming) - if (index === -1) return - setState((current) => ({ - ...current, - messages: current.messages.map((message, i) => - i === index && message.kind === "assistant" ? { ...message, streaming: false } : message, - ), - })) - redraw() + applyView({ type: "assistant.done" }) + }, + toolStart: (callID, name, input) => { + if ( + pendingTools.has(callID) || + view.messages.some((message) => message.kind === "tool" && message.callID === callID) + ) + return + const pending = { + name, + input, + completed: false, + showAt: performance.now() + TOOL_DISPLAY_DELAY_MS, + timer: setTimeout(() => { + const current = pendingTools.get(callID) + if (!current) return + pendingTools.delete(callID) + applyView({ + type: "tool.started", + callID, + name: current.name, + input: current.input, + }) + if (current.completed) applyView({ type: "tool.done", callID, output: current.output }) + }, TOOL_DISPLAY_DELAY_MS), + } + pendingTools.set(callID, pending) }, - toolStart: (callID, name, input) => push({ kind: "tool", callID, name, input }), toolDone: (callID, output) => { - const index = state().messages.findIndex((message) => message.kind === "tool" && message.callID === callID) - if (index === -1) return - setState((current) => ({ - ...current, - messages: current.messages.map((message, i) => - i === index && message.kind === "tool" ? { ...message, output } : message, - ), - })) - redraw() + const pending = pendingTools.get(callID) + if (pending) { + pending.output = output + pending.completed = true + return + } + applyView({ type: "tool.done", callID, output }) }, setStatus: (patch) => { - setState((current) => ({ ...current, status: { ...current.status, ...patch } })) - redraw() + if (connectedAt === undefined && patch.audio !== undefined) { + connectedAt = performance.now() + startAnimation() + } + setStatus((current) => ({ ...current, ...patch })) }, close: () => { if (!renderer.isDestroyed) renderer.destroy() diff --git a/packages/voice/src/voice-coordinator.ts b/packages/voice/src/voice-coordinator.ts new file mode 100644 index 0000000000..b923f085a5 --- /dev/null +++ b/packages/voice/src/voice-coordinator.ts @@ -0,0 +1,242 @@ +import type { CompletionReceipt } from "./opencode-notification" + +export type VoiceNotification = { + readonly id: string + readonly receipt?: CompletionReceipt + readonly promptID?: string + readonly delegationID?: string + readonly text: string +} + +export type VoiceState = { + readonly connection: "connecting" | "ready" | "closed" + readonly conversation: "idle" | "waiting" | "responding" + readonly assistant: "idle" | "active" | "suppressed" + readonly userSpeaking: boolean + readonly tools: ReadonlySet + readonly notifications: ReadonlyArray + readonly delivery?: { + readonly notification: VoiceNotification + readonly phase: "sending" | "accepted" | "announcing" | "awaiting-playback" + } + readonly desiredVoice: string + readonly reconnect: "none" | "pending" +} + +export type VoiceEvent = + | { readonly type: "connection.ready" } + | { readonly type: "connection.connecting" } + | { readonly type: "connection.closed" } + | { readonly type: "user.started"; readonly bargeIn?: boolean } + | { readonly type: "user.stopped" } + | { readonly type: "user.committed" } + | { readonly type: "assistant.started" } + | { readonly type: "assistant.suppressed" } + | { readonly type: "assistant.done"; readonly awaitingWork: boolean } + | { readonly type: "tool.started"; readonly id: string } + | { readonly type: "tool.finished"; readonly id: string } + | { readonly type: "notification.queued"; readonly notification: VoiceNotification } + | { readonly type: "notification.accepted"; readonly id: string } + | { readonly type: "notification.failed"; readonly id: string } + | { readonly type: "notification.announced"; readonly id: string } + | { readonly type: "notification.interrupted"; readonly id: string } + | { readonly type: "voice.selected"; readonly voice: string } + +export type VoiceCommand = + | { readonly type: "notification.send"; readonly notification: VoiceNotification } + | { readonly type: "notification.delivered"; readonly notification: VoiceNotification } + | { readonly type: "assistant.finish"; readonly awaitingWork: boolean; readonly notificationID?: string } + | { readonly type: "assistant.interrupt" } + | { readonly type: "connection.reconnect" } + +export type VoiceTransition = { + readonly state: VoiceState + readonly commands: ReadonlyArray +} + +export function initialVoiceState(voice: string): VoiceState { + return { + connection: "connecting", + conversation: "idle", + assistant: "idle", + userSpeaking: false, + tools: new Set(), + notifications: [], + desiredVoice: voice, + reconnect: "none", + } +} + +export function transitionVoice(state: VoiceState, event: VoiceEvent): VoiceTransition { + if (event.type === "notification.announced") { + if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "awaiting-playback") + return { state, commands: [] } + const notification = state.delivery.notification + const next = settle({ ...state, delivery: undefined }) + return { + state: next.state, + commands: [{ type: "notification.delivered", notification }, ...next.commands], + } + } + if (event.type === "assistant.done") { + const notificationID = + state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork + ? state.delivery.notification.id + : undefined + const next = settle(reduce(state, event)) + return { + state: next.state, + commands: [{ type: "assistant.finish", awaitingWork: event.awaitingWork, notificationID }, ...next.commands], + } + } + if (event.type === "user.started" && event.bargeIn && state.assistant === "active") { + const next = settle( + state.delivery + ? { + ...state, + conversation: "idle", + assistant: "idle", + userSpeaking: true, + delivery: undefined, + notifications: [state.delivery.notification, ...state.notifications], + reconnect: "pending", + } + : { ...state, assistant: "suppressed", userSpeaking: true }, + ) + return { state: next.state, commands: [{ type: "assistant.interrupt" }, ...next.commands] } + } + const next = reduce(state, event) + return settle(next) +} + +function reduce(state: VoiceState, event: VoiceEvent): VoiceState { + switch (event.type) { + case "connection.ready": + if (state.connection === "ready") return state + return { ...state, connection: "ready" } + case "connection.connecting": + return { + ...state, + connection: "connecting", + conversation: "idle", + assistant: "idle", + userSpeaking: false, + } + case "connection.closed": + return { ...state, connection: "closed", conversation: "idle", assistant: "idle", userSpeaking: false } + case "user.started": + if (state.userSpeaking) return state + return { ...state, userSpeaking: true } + case "user.stopped": + if (!state.userSpeaking) return state + return { ...state, userSpeaking: false } + case "user.committed": + return { + ...state, + userSpeaking: false, + conversation: state.assistant === "active" ? "responding" : "waiting", + assistant: state.assistant === "suppressed" ? "idle" : state.assistant, + } + case "assistant.started": + if (state.assistant === "suppressed" || state.assistant === "active") return state + return { + ...state, + conversation: "responding", + assistant: "active", + delivery: + state.delivery?.phase === "accepted" ? { ...state.delivery, phase: "announcing" } : state.delivery, + } + case "assistant.suppressed": + return { ...state, assistant: "suppressed" } + case "assistant.done": + return { + ...state, + conversation: event.awaitingWork ? "waiting" : "idle", + assistant: "idle", + delivery: + state.delivery?.phase === "announcing" && state.assistant === "active" && !event.awaitingWork + ? { ...state.delivery, phase: "awaiting-playback" } + : state.delivery, + } + case "tool.started": + if (state.tools.has(event.id)) return state + return { ...state, conversation: "waiting", tools: new Set([...state.tools, event.id]) } + case "tool.finished": { + if (!state.tools.has(event.id)) return state + const tools = new Set(state.tools) + tools.delete(event.id) + return { ...state, tools } + } + case "notification.queued": + return { ...state, notifications: [...state.notifications, event.notification] } + case "notification.accepted": + if (state.delivery?.notification.id !== event.id || state.delivery.phase !== "sending") return state + return { + ...state, + delivery: { ...state.delivery, phase: "accepted" }, + } + case "notification.failed": + if (state.delivery?.notification.id !== event.id) return state + return { + ...state, + connection: "ready", + conversation: "idle", + delivery: undefined, + notifications: [state.delivery.notification, ...state.notifications], + reconnect: "pending", + } + case "notification.announced": + return state + case "notification.interrupted": + if (state.delivery?.notification.id !== event.id) return state + return { + ...state, + conversation: "idle", + assistant: "idle", + delivery: undefined, + notifications: [state.delivery.notification, ...state.notifications], + reconnect: "pending", + } + case "voice.selected": + if (event.voice === state.desiredVoice) return state + return { ...state, desiredVoice: event.voice, reconnect: "pending" } + } + return state +} + +function settle(state: VoiceState): VoiceTransition { + if ( + state.reconnect === "pending" && + state.connection === "ready" && + state.conversation === "idle" && + state.delivery === undefined && + !state.userSpeaking && + state.tools.size === 0 + ) + return { + state: { ...state, connection: "connecting", reconnect: "none" }, + commands: [{ type: "connection.reconnect" }], + } + + if ( + state.notifications.length > 0 && + state.delivery === undefined && + state.connection === "ready" && + state.conversation === "idle" && + !state.userSpeaking && + state.tools.size === 0 + ) { + const notification = state.notifications[0] + return { + state: { + ...state, + conversation: "waiting", + delivery: { notification, phase: "sending" }, + notifications: state.notifications.slice(1), + }, + commands: [{ type: "notification.send", notification }], + } + } + + return { state, commands: [] } +} diff --git a/packages/voice/src/voice-session.ts b/packages/voice/src/voice-session.ts new file mode 100644 index 0000000000..82a7a556c5 --- /dev/null +++ b/packages/voice/src/voice-session.ts @@ -0,0 +1,446 @@ +import type { AudioEvent, AudioSession } from "./audio-session" +import { openCodeAnnouncementText, type CompletionReceipt, type OpenCodeAnnouncement } from "./opencode-notification" +import { createNotificationRouter } from "./notification-router" +import type { + VoiceConnection, + VoiceDelegationRequest, + VoiceProtocol, + VoiceProtocolEvent, + VoiceProtocolOptions, + VoiceTool, + VoiceToolExecution, + VoiceWorkRequest, +} from "./protocol" +import type { VoiceUI } from "./ui" +import { initialVoiceState, transitionVoice, type VoiceCommand, type VoiceEvent } from "./voice-coordinator" + +export type VoiceSession = { + start(): void + cycleVoice(): void + interrupt(): void + toggleMicrophone(): void + toggleSpeaker(): void + queueNotification(announcement: OpenCodeAnnouncement): void + close(): Promise +} + +export function voiceControlTool(voices: ReadonlyArray): VoiceTool { + return { + type: "function", + name: "set_voice", + description: "Change your speaking voice. Requires a brief reconnect and resets voice conversation memory.", + parameters: { + type: "object", + additionalProperties: false, + properties: { voice: { type: "string", enum: voices } }, + required: ["voice"], + }, + } +} + +export function createVoiceSession(options: { + readonly protocol: VoiceProtocol + readonly connection: Omit + readonly initialVoice: string + readonly voices: ReadonlyArray + readonly text?: string + readonly ui: VoiceUI + readonly audio?: AudioSession + readonly tools: { + readonly execute: (name: string, input: Record) => Promise + readonly acknowledge: (receipt: CompletionReceipt) => Promise + readonly close: () => Promise + } + readonly delegate: ( + request: VoiceDelegationRequest, + execute: (request: VoiceWorkRequest) => Promise, + ) => Promise + readonly trace?: (event: string, data?: Record) => void + readonly onClosed: () => void +}): VoiceSession { + let state = initialVoiceState(options.initialVoice) + let connection: VoiceConnection | undefined + let closed = false + let closePromise: Promise | undefined + let voiceTimer: ReturnType | undefined + const notifications = createNotificationRouter((routed) => { + options.trace?.("notification.queued", { + type: routed.announcement.notification.type, + promptID: routed.promptID, + depth: state.notifications.length + 1, + }) + dispatch({ + type: "notification.queued", + notification: { + id: crypto.randomUUID(), + receipt: routed.announcement.receipt, + promptID: routed.promptID, + delegationID: routed.delegationID, + text: openCodeAnnouncementText(routed.announcement.notification), + }, + }) + }) + + function dispatch(event: VoiceEvent) { + const transition = transitionVoice(state, event) + if (transition.state === state && transition.commands.length === 0) return + state = transition.state + options.trace?.("voice.transition", { + input: event.type, + connection: state.connection, + conversation: state.conversation, + assistant: state.assistant, + userSpeaking: state.userSpeaking, + tools: state.tools.size, + notifications: state.notifications.length, + commands: transition.commands.map((command) => command.type), + }) + transition.commands.forEach(runCommand) + } + + const setVoice = (voice: string) => { + options.ui.setStatus({ voice }) + options.ui.meta(`[voice] switching to ${voice}…`) + dispatch({ type: "voice.selected", voice }) + } + + const interruptDelivery = () => { + if (state.delivery) dispatch({ type: "notification.interrupted", id: state.delivery.notification.id }) + } + + const reconnect = () => { + if (closed) return + options.audio?.flushPlayback() + options.ui.assistantDone() + const previous = connection + connection = undefined + const next = () => { + if (!closed) connect() + } + if (!previous) return next() + void previous.close({ graceful: false }).finally(next) + } + + function runCommand(command: VoiceCommand) { + if (command.type === "connection.reconnect") return reconnect() + if (command.type === "assistant.interrupt") { + connection?.interrupt?.() + options.audio?.flushPlayback() + options.ui.assistantDone() + return + } + if (command.type === "assistant.finish") { + if (options.text) { + options.ui.assistantDone() + if (command.notificationID) dispatch({ type: "notification.announced", id: command.notificationID }) + if (!command.awaitingWork && state.tools.size === 0) options.onClosed() + return + } + void options.audio?.finishPlayback().then((outcome) => { + options.ui.assistantDone() + if (!command.notificationID) return + dispatch({ + type: outcome === "played" ? "notification.announced" : "notification.interrupted", + id: command.notificationID, + }) + }) + return + } + if (command.type === "notification.delivered") { + options.trace?.("notification.announced", { + id: command.notification.id, + promptID: command.notification.promptID, + }) + if (!command.notification.receipt) return + const receipt = command.notification.receipt + void options.tools + .acknowledge(receipt) + .catch((error) => + options.trace?.("notification.ack.failed", { id: command.notification.id, error: String(error) }), + ) + .finally(() => notifications.acknowledged(receipt)) + return + } + const active = connection + if (!active) return dispatch({ type: "notification.failed", id: command.notification.id }) + void active + .notify({ + id: command.notification.id, + text: `Announce this OpenCode update conversationally without reading private identifiers aloud:\n${command.notification.text}`, + delegationID: command.notification.delegationID, + }) + .then((accepted) => { + if (!accepted) return dispatch({ type: "notification.failed", id: command.notification.id }) + dispatch({ type: "notification.accepted", id: command.notification.id }) + options.trace?.("notification.context.accepted", { + id: command.notification.id, + promptID: command.notification.promptID, + remaining: state.notifications.length, + }) + }) + .catch((error) => { + options.trace?.("notification.context.failed", { + id: command.notification.id, + promptID: command.notification.promptID, + error: String(error), + }) + dispatch({ type: "notification.failed", id: command.notification.id }) + }) + } + + const executeTool = async (request: VoiceWorkRequest, delegationID?: string) => { + if (delegationID) notifications.beginDelegatedTool() + const execution: VoiceToolExecution = await (request.name === "set_voice" + ? Promise.resolve().then(() => { + const voice = request.input["voice"] + if (typeof voice !== "string" || !options.voices.includes(voice)) + return { output: toolError(`Voice must be one of: ${options.voices.join(", ")}.`) } + if (voiceTimer) clearTimeout(voiceTimer) + voiceTimer = setTimeout(() => { + voiceTimer = undefined + setVoice(voice) + }, 1_000) + return { + output: { + status: "switching", + voice, + note: `${options.protocol.name} conversation memory resets during reconnect.`, + }, + } + }) + : options.tools + .execute(request.name, request.input) + .catch((error) => ({ output: toolError(String(error), true) } satisfies VoiceToolExecution))) + options.ui.toolDone(request.id, execution.output) + if (delegationID) notifications.finishDelegatedTool(delegationID, execution.admittedPrompt) + return execution.output + } + + const queueWork = (source: VoiceConnection, request: VoiceWorkRequest) => { + options.trace?.("work.started", { id: request.id, name: request.name }) + void executeTool(request) + .then((output) => { + options.trace?.("work.resolved", { id: request.id, name: request.name }) + source.resolveWork(request, output) + }) + .catch((error) => options.ui.meta(`[work error] ${String(error)}`)) + .finally(() => dispatch({ type: "tool.finished", id: request.id })) + } + + const userSpeaking = (active: boolean) => { + if (active === state.userSpeaking) return + dispatch( + active + ? { type: "user.started", bargeIn: options.audio?.fullDuplex && options.protocol.capabilities.interruption } + : { type: "user.stopped" }, + ) + options.ui.userSpeaking(active) + } + + const onAudioEvent = (event: AudioEvent) => { + switch (event.type) { + case "input": + connection?.appendAudio(event.audio) + options.ui.userAudioLevel(event.level) + return + case "user.started": + userSpeaking(true) + return + case "user.stopped": + userSpeaking(false) + return + case "user.reset": + options.ui.userReset() + return + case "assistant.level": + options.ui.assistantAudio(event.level, event.durationMs) + return + case "status": + options.ui.setStatus({ audio: event.audio }) + return + case "meta": + options.ui.meta(event.text) + } + } + + const onProtocolEvent = (source: VoiceConnection, event: VoiceProtocolEvent) => { + if (source !== connection || closed) return + if (options.connection.debug || event.type !== "assistant.audio") + options.trace?.("protocol.event", { + type: event.type, + conversation: state.conversation, + assistant: state.assistant, + userSpeaking: state.userSpeaking, + pendingWork: state.tools.size, + }) + switch (event.type) { + case "ready": + dispatch({ type: "connection.ready" }) + options.ui.meta( + `connected to ${options.protocol.name} ${options.connection.model} (voice: ${state.desiredVoice})`, + ) + options.ui.setStatus({ + voice: state.desiredVoice, + microphoneMuted: options.audio?.microphoneMuted, + speakerMuted: options.audio?.speakerMuted, + }) + if (!options.text) { + void options.audio?.start(onAudioEvent) + return + } + options.ui.userTranscript("typed", options.text) + dispatch({ type: "user.committed" }) + source.sendText?.(options.text) + return + case "user.started": + userSpeaking(true) + return + case "user.stopped": + userSpeaking(false) + return + case "user.committed": + options.audio?.noteUserCommitted() + dispatch({ type: "user.committed" }) + options.ui.userCommitted(event.id) + return + case "user.transcript": + options.audio?.noteUserTranscript(event.final) + if (event.final) { + dispatch({ type: "user.stopped" }) + options.ui.userSpeaking(false) + } + options.ui.userTranscript(event.id, event.text, event.final) + return + case "assistant.audio": + if (state.assistant === "suppressed") return + dispatch({ type: "assistant.started" }) + options.audio?.play(event.audio, event.timeline) + return + case "assistant.transcript.delta": + if (state.assistant === "suppressed") return + dispatch({ type: "assistant.started" }) + options.ui.assistantDelta(event.delta) + return + case "assistant.transcript": + if (state.assistant === "suppressed") return + dispatch({ type: "assistant.started" }) + options.ui.assistantTranscript(event.text) + return + case "assistant.done": { + dispatch({ type: "assistant.done", awaitingWork: event.awaitingWork }) + return + } + case "tool.started": + dispatch({ type: "tool.started", id: event.id }) + options.ui.toolStart(event.id, event.name, {}) + return + case "work.requested": + queueWork(source, event.request) + return + case "work.rejected": + dispatch({ type: "tool.started", id: event.request.id }) + options.ui.toolStart(event.request.id, event.request.name, event.request.input) + options.ui.toolDone(event.request.id, event.output) + source.resolveWork(event.request, event.output) + dispatch({ type: "tool.finished", id: event.request.id }) + return + case "delegation.requested": + dispatch({ type: "tool.started", id: event.request.id }) + void options + .delegate(event.request, (request) => { + options.ui.toolStart(request.id, request.name, request.input) + return executeTool(request, event.request.id) + }) + .then((output) => { + if (source !== connection) { + options.trace?.("delegation.result.stale", { id: event.request.id }) + return + } + source.resolveDelegation?.(event.request, output) + }) + .catch((error) => { + options.trace?.("responses.controller.failed", { error: String(error) }) + if (source !== connection) return + options.ui.toolStart(event.request.id, "opencode", { text: event.request.text }) + options.ui.toolDone(event.request.id, { status: "error", message: String(error) }) + source.resolveDelegation?.(event.request, `The OpenCode controller failed: ${String(error)}`) + }) + .finally(() => dispatch({ type: "tool.finished", id: event.request.id })) + return + case "debug": + if (options.connection.debug) options.ui.meta(`[debug] ${event.message}`) + return + case "error": + options.ui.meta(`[${options.protocol.name} error] ${event.message}`) + return + case "closed": + dispatch({ type: "connection.closed" }) + options.ui.meta(`${options.protocol.name} connection closed (${event.code})`) + options.onClosed() + } + } + + const connect = () => { + let next: VoiceConnection + next = options.protocol.connect({ + ...options.connection, + voice: state.desiredVoice, + onEvent: (event) => onProtocolEvent(next, event), + }) + dispatch({ type: "connection.connecting" }) + connection = next + } + + const session: VoiceSession = { + start: connect, + cycleVoice() { + setVoice(options.voices[(options.voices.indexOf(state.desiredVoice) + 1) % options.voices.length]) + }, + interrupt() { + if (!options.audio?.isPlaying() && state.assistant === "idle") return + if (state.assistant === "active") { + connection?.interrupt?.() + dispatch({ type: "assistant.suppressed" }) + } + interruptDelivery() + options.audio?.flushPlayback() + options.ui.assistantDone() + options.ui.meta("[interrupted]") + }, + toggleMicrophone() { + if (!options.audio) return + const muted = options.audio.toggleMicrophone() + userSpeaking(false) + options.ui.setStatus({ microphoneMuted: muted }) + if (muted) options.ui.userReset() + options.ui.userAudioLevel(undefined) + options.ui.meta(`[microphone] ${muted ? "muted" : "live"}`) + }, + toggleSpeaker() { + if (!options.audio) return + const muted = options.audio.toggleSpeaker() + options.ui.setStatus({ speakerMuted: muted }) + if (muted) options.ui.assistantPlaybackStopped() + options.ui.meta(`[speaker] ${muted ? "muted" : "live"}`) + }, + queueNotification(announcement) { + notifications.route(announcement) + }, + close() { + if (closePromise) return closePromise + closed = true + if (voiceTimer) clearTimeout(voiceTimer) + options.audio?.close() + const active = connection + connection = undefined + dispatch({ type: "connection.closed" }) + closePromise = Promise.allSettled([options.tools.close(), ...(active ? [active.close()] : [])]).then(() => {}) + return closePromise + }, + } + return session +} + +function toolError(message: string, retryable = false) { + return { status: "error", message, retryable } +} diff --git a/packages/voice/test/animation.test.ts b/packages/voice/test/animation.test.ts new file mode 100644 index 0000000000..c9c620a751 --- /dev/null +++ b/packages/voice/test/animation.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, test } from "bun:test" +import { + connectionMeterLevels, + connectionTransitioning, + scheduleTextReveal, + textRevealOpacity, + transcriptionPulse, +} from "../src/animation" + +describe("voice text animation", () => { + test("stages newly streamed words without delaying them indefinitely", () => { + const first = scheduleTextReveal("", "Hello there", 1_000, 0) + expect(first.reveals).toEqual([ + { offset: 0, at: 1_000 }, + { offset: 6, at: 1_032 }, + ]) + expect(first.animationEndsAt).toBe(1_412) + + const queued = scheduleTextReveal("Hello there", " friend", 1_010, 10_000) + expect(queued.reveals).toEqual([{ offset: 12, at: 1_170 }]) + }) + + test("slowly fades text into its stable color", () => { + const opacities = [100, 195, 290, 385, 480].map((now) => textRevealOpacity(now, 100)) + expect(opacities[0]).toBe(0.16) + expect(opacities.toSorted((a, b) => a - b)).toEqual(opacities) + expect(opacities.at(-1)).toBe(1) + }) + + test("keeps transcription pulse bounded and moving", () => { + const levels = [0, 110, 220, 330].map(transcriptionPulse) + expect(levels.every((level) => level >= 0 && level <= 1)).toBe(true) + expect(new Set(levels).size).toBeGreaterThan(1) + }) + + test("moves the connecting pulse left to right and blends into the connected meter", () => { + const first = connectionMeterLevels(1_000, 0.1, undefined, 1_000) + const next = connectionMeterLevels(1_180, 0.1, undefined, 1_000) + expect(first.indexOf(Math.max(...first))).toBe(0) + expect(first.indexOf(Math.max(...first))).toBeLessThan(next.indexOf(Math.max(...next))) + + const samples = Array.from({ length: 16 }, (_, index) => connectionMeterLevels(index * 45, 0.1)) + expect(samples.every((levels) => Math.max(...levels) > 0.85)).toBe(true) + + const handoff = connectionMeterLevels(1_000, 0.4, 1_000) + expect(handoff).toEqual(connectionMeterLevels(1_000, 0.4)) + expect(connectionMeterLevels(1_480, 0.4, 1_000)).toEqual( + [0, 1, 2, 3].map((index) => 0.4 * (0.72 + Math.sin(1_480 / 240 + index * 1.4) * 0.28)), + ) + expect(connectionTransitioning(1_479, 1_000)).toBe(true) + expect(connectionTransitioning(1_480, 1_000)).toBe(false) + }) +}) diff --git a/packages/voice/test/assistant-text.test.ts b/packages/voice/test/assistant-text.test.ts new file mode 100644 index 0000000000..dacbd61620 --- /dev/null +++ b/packages/voice/test/assistant-text.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, test } from "bun:test" +import { joinAssistantText } from "../src/ui-model" + +describe("joinAssistantText", () => { + const cases: Array<[name: string, left: string, right: string, expected: string]> = [ + [ + "inserts a space between sentences", + "Okay, I'll check that.", + "The build passed.", + "Okay, I'll check that. The build passed.", + ], + ["inserts a space mid-sentence", "Let me look at", "the config file.", "Let me look at the config file."], + [ + "keeps a single trailing space", + "Okay, I'll check that. ", + "The build passed.", + "Okay, I'll check that. The build passed.", + ], + [ + "keeps a single leading space", + "Okay, I'll check that.", + " The build passed.", + "Okay, I'll check that. The build passed.", + ], + ["collapses spaces on both sides", "Done. ", " Next.", "Done. Next."], + ["collapses runs of spaces", "Done. ", "\t \tNext.", "Done. Next."], + [ + "spaces after a question mark", + "Want me to run tests?", + "I can do that now.", + "Want me to run tests? I can do that now.", + ], + ["spaces after an ellipsis", "Thinking…", "done.", "Thinking… done."], + ["preserves a trailing newline", "Done:\n", "Next up, tests.", "Done:\nNext up, tests."], + ["preserves a leading newline", "Done:", "\nNext up, tests.", "Done:\nNext up, tests."], + [ + "preserves a paragraph break", + "First paragraph.\n\n", + "Second paragraph.", + "First paragraph.\n\nSecond paragraph.", + ], + ["preserves a newline mixed with spaces", "Done: ", " \n Next.", "Done: \n Next."], + ["never spaces before closing punctuation", "Done", ".", "Done."], + ["never spaces before a comma", "one", ", two", "one, two"], + ["never spaces before a closing paren", "(aside", ")", "(aside)"], + ["never spaces after an opening paren", "an (", "aside)", "an (aside)"], + ["drops an empty left side", "", "The build passed.", "The build passed."], + ["drops an empty right side", "The build passed.", "", "The build passed."], + ["drops a whitespace-only left side", " ", "The build passed.", "The build passed."], + ["drops a whitespace-only right side", "The build passed.", " \n ", "The build passed."], + ["handles two empty sides", "", "", ""], + ["does not trim the outer edges", " Leading kept.", "Trailing kept. ", " Leading kept. Trailing kept. "], + ] + + for (const [name, left, right, expected] of cases) { + test(name, () => expect(joinAssistantText(left, right).text).toBe(expected)) + } + + test("splits the result at the boundary so reveals cover only the appended text", () => { + for (const [name, left, right] of cases) { + const joined = joinAssistantText(left, right) + expect(`${name}: ${joined.previous}${joined.appended}`).toBe(`${name}: ${joined.text}`) + expect(`${name}: ${joined.text.endsWith(joined.appended)}`).toBe(`${name}: true`) + } + }) + + test("is idempotent once a boundary is normalised", () => { + const once = joinAssistantText("Okay.", "Next.").text + expect(joinAssistantText(once, "").text).toBe(once) + expect(joinAssistantText("Okay. ", "Next.").text).toBe(once) + }) + + test("folds consecutive messages with exactly one space per boundary", () => { + expect(["First.", "Second.", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe( + "First. Second. Third.", + ) + expect(["First.", "", "Third."].reduce((left, right) => joinAssistantText(left, right).text)).toBe("First. Third.") + }) +}) diff --git a/packages/voice/test/audio-jitter-buffer.test.ts b/packages/voice/test/audio-jitter-buffer.test.ts new file mode 100644 index 0000000000..4896ccb7c5 --- /dev/null +++ b/packages/voice/test/audio-jitter-buffer.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from "bun:test" +import { AudioJitterBuffer } from "../src/audio-jitter-buffer" +import { PCM_BYTES_PER_MS } from "../src/pcm" + +const chunk = (durationMs: number, gapMs = 0) => ({ bytes: Buffer.alloc(durationMs * PCM_BYTES_PER_MS), gapMs }) + +describe("AudioJitterBuffer", () => { + test("holds startup audio until the target buffer is available", () => { + const buffer = new AudioJitterBuffer(300) + expect(buffer.push(chunk(100))).toEqual([]) + expect(buffer.push(chunk(100))).toEqual([]) + expect(buffer.push(chunk(100))).toHaveLength(3) + }) + + test("passes chunks through after playback starts", () => { + const buffer = new AudioJitterBuffer(200) + buffer.push(chunk(200)) + expect(buffer.push(chunk(100))).toEqual([chunk(100)]) + }) + + test("flushes short utterances and resets between responses", () => { + const buffer = new AudioJitterBuffer(300) + buffer.push(chunk(100)) + expect(buffer.finish()).toHaveLength(1) + buffer.reset() + expect(buffer.push(chunk(100))).toEqual([]) + }) + + test("counts intentional timeline silence toward buffered duration", () => { + const buffer = new AudioJitterBuffer(300) + expect(buffer.push(chunk(200, 100))).toHaveLength(1) + }) +}) diff --git a/packages/voice/test/audio-session.test.ts b/packages/voice/test/audio-session.test.ts new file mode 100644 index 0000000000..773afd6e2d --- /dev/null +++ b/packages/voice/test/audio-session.test.ts @@ -0,0 +1,106 @@ +import { expect, test } from "bun:test" +import type { AudioDevice } from "../src/audio-device" +import { createAudioSession, type AudioEvent } from "../src/audio-session" + +test("reports muted notification playback as inaudible", async () => { + const device = scriptedDevice() + const audio = await createAudioSession({ + duplex: false, + speakers: false, + inputActivity: "server", + debug: false, + device, + }) + + expect(audio.toggleSpeaker()).toBe(true) + audio.play(Buffer.alloc(4_800)) + expect(await audio.finishPlayback()).toBe("inaudible") + expect(device.writes).toEqual([]) +}) + +test("settles active playback when interrupted", async () => { + const device = scriptedDevice() + const audio = await createAudioSession({ + duplex: false, + speakers: false, + inputActivity: "server", + debug: false, + device, + }) + + audio.play(Buffer.alloc(4_800)) + const outcome = audio.finishPlayback() + audio.flushPlayback() + expect(await outcome).toBe("interrupted") + expect(device.flushes).toBe(1) +}) + +test("emits captured audio and local speech observations", async () => { + const device = scriptedDevice() + const events: AudioEvent[] = [] + const audio = await createAudioSession({ + duplex: true, + speakers: false, + inputActivity: "local", + debug: false, + device, + }) + await audio.start((event) => events.push(event)) + const input = Buffer.alloc(4_800) + for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(12_000, offset) + device.capture(input) + + expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) }) + expect(events).toContainEqual({ type: "user.started" }) + audio.close() +}) + +test("preserves quiet duplex microphone audio during assistant playback", async () => { + const device = scriptedDevice() + const events: AudioEvent[] = [] + const audio = await createAudioSession({ + duplex: false, + speakers: true, + inputActivity: "server", + debug: false, + device, + }) + await audio.start((event) => events.push(event)) + audio.play(Buffer.alloc(24_000)) + const input = Buffer.alloc(4_800) + for (let offset = 0; offset < input.length; offset += 2) input.writeInt16LE(800, offset) + device.capture(input) + + expect(audio.fullDuplex).toBe(true) + expect(events).toContainEqual({ type: "input", audio: input, level: expect.any(Number) }) + audio.close() +}) + +function scriptedDevice() { + let onInput: ((audio: Buffer) => void) | undefined + const writes: Buffer[] = [] + const device = { + fullDuplex: true, + mode: "scripted", + async start(listener) { + onInput = listener + }, + write(audio) { + writes.push(audio) + }, + flush() { + device.flushes += 1 + }, + close() {}, + writes, + flushes: 0, + capture(audio: Buffer) { + onInput?.(audio) + }, + } satisfies AudioDevice & { + readonly writes: Buffer[] + flushes: number + capture(audio: Buffer): void + } + return device +} diff --git a/packages/voice/test/completion-store.test.ts b/packages/voice/test/completion-store.test.ts new file mode 100644 index 0000000000..fcca61d8a7 --- /dev/null +++ b/packages/voice/test/completion-store.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from "bun:test" +import { mkdir, mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createCompletionStore } from "../src/completion-store" + +test("completion store restores pending and completed notifications until delivery", async () => { + const directory = await mkdtemp(join(tmpdir(), "voice-completions-")) + const path = join(directory, "prompts.json") + const first = await createCompletionStore(path) + const pending = { sessionID: "session-1", promptID: "prompt-1" } + const completed = { sessionID: "session-2", promptID: "prompt-2" } + + try { + await first.admitting(pending, "Please continue") + expect(first.entries()).toEqual([{ status: "admitting", handle: pending, text: "Please continue" }]) + await first.pending(pending) + await first.pending(completed) + await first.completed(completed, { + type: "opencode.prompt.completed", + session_id: "session-2", + prompt_id: "prompt-2", + status: "completed", + text: "done", + }) + await first.close() + + const restored = await createCompletionStore(path) + expect(restored.entries()).toEqual(expect.arrayContaining([ + { status: "pending", handle: pending }, + { + status: "completed", + handle: completed, + notification: { + type: "opencode.prompt.completed", + session_id: "session-2", + prompt_id: "prompt-2", + status: "completed", + text: "done", + }, + }, + ])) + await restored.close() + + const stillPendingDelivery = await createCompletionStore(path) + expect(stillPendingDelivery.entries().some((entry) => entry.status === "completed")).toBe(true) + + await stillPendingDelivery.remove(completed) + await stillPendingDelivery.close() + expect((await createCompletionStore(path)).entries()).toEqual([{ status: "pending", handle: pending }]) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test("completion stores do not overwrite prompts written by another process", async () => { + const directory = await mkdtemp(join(tmpdir(), "voice-completions-concurrent-")) + const path = join(directory, "prompts.json") + const first = await createCompletionStore(path) + const second = await createCompletionStore(path) + + try { + await Promise.all([ + first.pending({ sessionID: "session-1", promptID: "prompt-shared" }), + second.pending({ sessionID: "session-2", promptID: "prompt-shared" }), + ]) + await Promise.all([first.close(), second.close()]) + expect((await createCompletionStore(path)).entries()).toEqual( + expect.arrayContaining([ + { status: "pending", handle: { sessionID: "session-1", promptID: "prompt-shared" } }, + { status: "pending", handle: { sessionID: "session-2", promptID: "prompt-shared" } }, + ]), + ) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test("continues serializing writes after one filesystem failure", async () => { + const directory = await mkdtemp(join(tmpdir(), "voice-completions-recovery-")) + const path = join(directory, "prompts.json") + const store = await createCompletionStore(path) + try { + await rm(`${path}.d`, { recursive: true }) + await Bun.write(`${path}.d`, "not a directory") + const failure = await store + .pending({ sessionID: "session-1", promptID: "prompt-1" }) + .then(() => undefined, (error) => error) + expect(failure).toBeDefined() + await rm(`${path}.d`) + await mkdir(`${path}.d`, { recursive: true }) + await store.pending({ sessionID: "session-2", promptID: "prompt-2" }) + await store.close() + expect((await createCompletionStore(path)).entries()).toContainEqual({ + status: "pending", + handle: { sessionID: "session-2", promptID: "prompt-2" }, + }) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) diff --git a/packages/voice/test/live-projector.test.ts b/packages/voice/test/live-projector.test.ts new file mode 100644 index 0000000000..214c304c86 --- /dev/null +++ b/packages/voice/test/live-projector.test.ts @@ -0,0 +1,311 @@ +import { describe, expect, test } from "bun:test" +import { createLiveEventDelivery, createLiveEventProjector } from "../src/protocol-live" +import type { VoiceProtocolEvent } from "../src/protocol" + +const frames = [ + { type: "session.started", session: {} }, + { type: "session.context.appended", start_ms: 0, end_ms: 0 }, + { type: "output_audio.delta", audio: "AAA=", start_ms: 0, end_ms: 1 }, + { + type: "input_transcript.added", + start_ms: 0, + end_ms: 100, + item: { id: "input-1", type: "input_transcript", text: " Hello" }, + }, + { type: "turn.created", turn: { id: "user-turn", role: "user", transcript: " Hello" } }, + { + type: "input_transcript.added", + start_ms: 100, + end_ms: 200, + item: { id: "input-2", type: "input_transcript", text: ", this" }, + }, + { type: "turn.delta", turn_id: "user-turn", delta: ", this" }, + { + type: "output_transcript.added", + start_ms: 200, + end_ms: 300, + item: { id: "output-1", type: "output_transcript", text: " Copy that" }, + }, + { + type: "input_transcript.added", + start_ms: 200, + end_ms: 300, + item: { id: "input-3", type: "input_transcript", text: " is a test" }, + }, + { + type: "output_transcript.added", + start_ms: 300, + end_ms: 400, + item: { id: "output-2", type: "output_transcript", text: ", test" }, + }, + { type: "turn.delta", turn_id: "user-turn", delta: " is a test" }, + { type: "turn.done", turn: { id: "user-turn", role: "user", transcript: " Hello, this is a test" } }, + { type: "turn.created", turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test" } }, + { + type: "output_transcript.added", + start_ms: 400, + end_ms: 500, + item: { id: "output-3", type: "output_transcript", text: " received" }, + }, + { type: "turn.delta", turn_id: "assistant-turn", delta: " received" }, + { + type: "output_transcript.added", + start_ms: 500, + end_ms: 600, + item: { id: "output-4", type: "output_transcript", text: " loud and clear." }, + }, + { type: "turn.delta", turn_id: "assistant-turn", delta: " loud and clear." }, + { + type: "turn.done", + turn: { id: "assistant-turn", role: "assistant", transcript: " Copy that, test received loud and clear." }, + }, +] as const + +describe("Live event projector", () => { + test("replays progressive transcripts without schema errors or duplicate text", () => { + const project = createLiveEventProjector() + const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events) + + expect(events.filter((event) => event.type === "error")).toEqual([]) + expect(events.filter((event) => event.type === "user.committed")).toEqual([ + { type: "user.committed", id: "input-1" }, + ]) + expect(events.filter(isUserTranscript)).toEqual([ + { type: "user.transcript", id: "input-1", text: "Hello", final: false }, + { type: "user.transcript", id: "input-1", text: "Hello, this", final: false }, + { type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: false }, + { type: "user.transcript", id: "input-1", text: "Hello, this is a test", final: true }, + ]) + expect( + events + .filter(isAssistantDelta) + .map((event) => event.delta) + .join(""), + ).toBe(" Copy that, test received loud and clear.") + expect(events.filter((event) => event.type === "assistant.done")).toEqual([ + { type: "assistant.done", awaitingWork: false }, + ]) + expect(events.filter((event) => event.type === "assistant.audio")).toHaveLength(1) + }) + + test("reports malformed frames instead of throwing", () => { + expect(createLiveEventProjector()(JSON.stringify({ type: "turn.created", turn: { id: 1 } })).events).toEqual([ + { type: "error", message: "Received an invalid Live API event." }, + ]) + }) + + test("preserves context acknowledgement correlation IDs", () => { + expect( + createLiveEventProjector()( + JSON.stringify({ type: "session.context.appended", event_id: "event-notification" }), + ), + ).toMatchObject({ + type: "session.context.appended", + eventID: "event-notification", + }) + }) + + test("replaces corrected assistant transcript snapshots", () => { + const project = createLiveEventProjector() + const events = [ + project( + JSON.stringify({ + type: "output_transcript.added", + item: { id: "output-1", type: "output_transcript", text: " Hello world" }, + }), + ), + project( + JSON.stringify({ + type: "turn.created", + turn: { id: "assistant-turn", role: "assistant", transcript: " Hello there" }, + }), + ), + ].flatMap((result) => result.events) + + expect(events.filter((event) => event.type.startsWith("assistant.transcript"))).toEqual([ + { type: "assistant.transcript.delta", delta: " Hello world" }, + { type: "assistant.transcript", text: " Hello there" }, + ]) + }) + + test("keeps the space between consecutive assistant turns", () => { + const project = createLiveEventProjector() + const turn = (id: string, text: string) => [ + { type: "turn.created", turn: { id, role: "assistant", transcript: "" } }, + { type: "output_transcript.added", item: { id: `${id}-o`, type: "output_transcript", text } }, + { type: "turn.delta", turn_id: id, delta: text }, + { type: "turn.done", turn: { id, role: "assistant", transcript: text } }, + ] + const events = [...turn("t1", ' Session "Fix auth bug" is ready.'), ...turn("t2", " Next up, tests.")].flatMap( + (frame) => project(JSON.stringify(frame)).events, + ) + + // Concatenating the delta stream is what the UI and any transcript consumer does; the + // turn boundary must survive it rather than welding "ready.Next" together. + const transcript = events + .filter(isAssistantDelta) + .map((event) => event.delta) + .join("") + expect(transcript).toBe(' Session "Fix auth bug" is ready. Next up, tests.') + expect(transcript).not.toContain("ready.Next") + expect(events.filter((event) => event.type === "assistant.done")).toHaveLength(2) + }) + + test("emits no duplicate text when fragments and turn deltas overlap across turns", () => { + const project = createLiveEventProjector() + const frames = [ + { type: "turn.created", turn: { id: "t1", role: "assistant", transcript: "" } }, + { type: "output_transcript.added", item: { id: "a", type: "output_transcript", text: " First" } }, + { type: "turn.delta", turn_id: "t1", delta: " First" }, + { type: "output_transcript.added", item: { id: "b", type: "output_transcript", text: " turn." } }, + { type: "turn.delta", turn_id: "t1", delta: " turn." }, + { type: "turn.done", turn: { id: "t1", role: "assistant", transcript: " First turn." } }, + { type: "turn.created", turn: { id: "t2", role: "assistant", transcript: "" } }, + { type: "output_transcript.added", item: { id: "c", type: "output_transcript", text: " Second turn." } }, + { type: "turn.delta", turn_id: "t2", delta: " Second turn." }, + { type: "turn.done", turn: { id: "t2", role: "assistant", transcript: " Second turn." } }, + ] + const events = frames.flatMap((frame) => project(JSON.stringify(frame)).events) + + expect(events.filter((event) => event.type === "assistant.transcript")).toEqual([]) + expect( + events + .filter(isAssistantDelta) + .map((event) => event.delta) + .join(""), + ).toBe(" First turn. Second turn.") + }) + + test("projects Responses delegation function calls without rejecting reasoning items", () => { + const project = createLiveEventProjector() + expect( + project( + JSON.stringify({ + type: "response.output_item.done", + item: { id: "reasoning-1", type: "reasoning" }, + }), + ).events, + ).toEqual([{ type: "debug", message: "response.output_item.done" }]) + expect( + project( + JSON.stringify({ + type: "response.output_item.added", + item: { + id: "function-1", + type: "function_call", + name: "find_sessions", + call_id: "call-1", + arguments: "", + }, + }), + ).events, + ).toEqual([ + { type: "debug", message: "response.output_item.added" }, + { type: "tool.started", id: "call-1", name: "find_sessions" }, + ]) + expect( + project( + JSON.stringify({ + type: "response.output_item.done", + item: { + id: "function-1", + type: "function_call", + name: "find_sessions", + call_id: "call-1", + arguments: '{"scope":"current_project"}', + }, + }), + ).events, + ).toEqual([ + { type: "debug", message: "response.output_item.done" }, + { + type: "work.requested", + request: { id: "call-1", name: "find_sessions", input: { scope: "current_project" } }, + }, + ]) + expect( + project( + JSON.stringify({ + type: "response.output_item.done", + item: { + id: "message-1", + type: "message", + status: "completed", + content: [{ type: "output_text", text: "Two sessions found." }], + role: "assistant", + }, + }), + ).events, + ).toEqual([{ type: "debug", message: "response.output_item.done" }]) + }) + + test("projects client delegation requests for the local controller", () => { + expect( + createLiveEventProjector()( + JSON.stringify({ + type: "delegation.created", + item: { + id: "delegation-1", + type: "delegation", + target: "client", + content: [{ type: "input_text", text: "Inspect the voice package" }], + }, + }), + ).events, + ).toEqual([ + { type: "debug", message: "delegation.created" }, + { type: "delegation.requested", request: { id: "delegation-1", text: "Inspect the voice package" } }, + ]) + }) + + test("rejects malformed function arguments without hanging the delegation", () => { + expect( + createLiveEventProjector()( + JSON.stringify({ + type: "response.output_item.done", + item: { + id: "function-1", + type: "function_call", + name: "find_sessions", + call_id: "call-invalid", + arguments: "not json", + }, + }), + ).events, + ).toEqual([ + { type: "debug", message: "response.output_item.done" }, + { type: "tool.started", id: "call-invalid", name: "find_sessions" }, + { type: "error", message: "Received invalid arguments for Live tool find_sessions." }, + { + type: "work.rejected", + request: { id: "call-invalid", name: "find_sessions", input: {} }, + output: { status: "error", message: "Invalid arguments for tool find_sessions." }, + }, + ]) + }) + + test("waits for a quiet drain after turn completion and late audio", async () => { + const events: VoiceProtocolEvent[] = [] + const delivery = createLiveEventDelivery({ emit: (event) => events.push(event), drainMs: 20 }) + delivery.push({ type: "assistant.done", awaitingWork: false }) + await Bun.sleep(10) + delivery.push({ type: "assistant.audio", audio: Buffer.alloc(48) }) + await Bun.sleep(12) + expect(events.map((event) => event.type)).toEqual(["assistant.audio"]) + await Bun.sleep(12) + expect(events.map((event) => event.type)).toEqual(["assistant.audio", "assistant.done"]) + delivery.close() + }) +}) + +function isUserTranscript( + event: VoiceProtocolEvent, +): event is Extract { + return event.type === "user.transcript" +} + +function isAssistantDelta( + event: VoiceProtocolEvent, +): event is Extract { + return event.type === "assistant.transcript.delta" +} diff --git a/packages/voice/test/notification-router.test.ts b/packages/voice/test/notification-router.test.ts new file mode 100644 index 0000000000..995d308867 --- /dev/null +++ b/packages/voice/test/notification-router.test.ts @@ -0,0 +1,59 @@ +import { expect, test } from "bun:test" +import { createNotificationRouter, type RoutedAnnouncement } from "../src/notification-router" + +test("routes blockers and completions through the admitted delegation", () => { + const routed: RoutedAnnouncement[] = [] + const router = createNotificationRouter((announcement) => routed.push(announcement)) + router.beginDelegatedTool() + router.route({ + notification: { + type: "opencode.prompt.blocked", + prompt_id: "prompt-1", + blocker: "permission", + session_id: "session-1", + request_id: "permission-1", + action: "read", + resources: [], + }, + }) + expect(routed).toEqual([]) + + router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-1" }) + router.route({ + notification: { + type: "opencode.prompt.completed", + session_id: "session-1", + prompt_id: "prompt-1", + status: "completed", + text: "done", + }, + receipt: { sessionID: "session-1", promptID: "prompt-1" }, + }) + + expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-1"]) + expect(routed[0].announcement.receipt).toBeUndefined() + expect(routed[1].announcement.receipt).toEqual({ sessionID: "session-1", promptID: "prompt-1" }) +}) + +test("keeps identical prompt IDs isolated by session", () => { + const routed: RoutedAnnouncement[] = [] + const router = createNotificationRouter((announcement) => routed.push(announcement)) + router.beginDelegatedTool() + router.finishDelegatedTool("delegation-1", { sessionID: "session-1", promptID: "prompt-shared" }) + router.beginDelegatedTool() + router.finishDelegatedTool("delegation-2", { sessionID: "session-2", promptID: "prompt-shared" }) + + for (const sessionID of ["session-1", "session-2"]) + router.route({ + notification: { + type: "opencode.prompt.completed", + session_id: sessionID, + prompt_id: "prompt-shared", + status: "completed", + text: "done", + }, + receipt: { sessionID, promptID: "prompt-shared" }, + }) + + expect(routed.map((item) => item.delegationID)).toEqual(["delegation-1", "delegation-2"]) +}) diff --git a/packages/voice/test/opencode.test.ts b/packages/voice/test/opencode.test.ts new file mode 100644 index 0000000000..3cd07e84d8 --- /dev/null +++ b/packages/voice/test/opencode.test.ts @@ -0,0 +1,248 @@ +import { expect, test } from "bun:test" +import { OpenCode } from "@opencode-ai/client/promise" +import { createOpenCodeBridge } from "../src/opencode" +import type { CompletionStore } from "../src/completion-store" + +test("renames only a session discovered by the voice controller", async () => { + const requests: Request[] = [] + const session = { + id: "ses_known", + projectID: "project-1", + cost: "0", + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 2 }, + title: "Old title", + location: { directory: "/workspace" }, + } + const fetch = Object.assign( + async (input: string | URL | Request, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input.toString(), init) + requests.push(request) + const url = new URL(request.url) + if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } }) + if (url.pathname === "/api/project") + return Response.json([ + { + id: "project-1", + worktree: "/workspace", + time: { created: 1, updated: 2 }, + sandboxes: [], + }, + ]) + if (url.pathname === "/api/session" && request.method === "GET") + return Response.json({ data: [session], cursor: {} }) + if (url.pathname === "/api/session/ses_known" && request.method === "GET") return Response.json({ data: session }) + if (url.pathname === "/api/session/ses_known/rename" && request.method === "POST") + return new Response(null, { status: 204 }) + return new Response("Not found", { status: 404 }) + }, + { preconnect: () => {} }, + ) + const client = OpenCode.make({ + baseUrl: "https://opencode.test", + fetch, + }) + const bridge = await createOpenCodeBridge({ + client, + directory: "/workspace", + model: { providerID: "openai", id: "gpt-test" }, + notify: () => {}, + completionStore: memoryCompletionStore(), + }) + const execute = async (name: string, input: Record) => (await bridge.execute(name, input)).output + + try { + expect(new Set(bridge.definitions.map((tool) => tool.name)).size).toBe(bridge.definitions.length) + expect(bridge.definitions.every((tool) => tool.name.length > 0 && tool.description.length > 0)).toBe(true) + expect(bridge.definitions.find((tool) => tool.name === "reply_form")).toMatchObject({ + parameters: { + properties: { answer: { type: "object", additionalProperties: true } }, + }, + }) + expect(bridge.definitions.find((tool) => tool.name === "rename_session")).toMatchObject({ + description: expect.stringContaining("does not prompt, wake, or interrupt"), + parameters: { + type: "object", + additionalProperties: false, + required: ["session_id", "title"], + }, + }) + + expect(await execute("rename_session", { session_id: "ses_unknown", title: "Nope" })).toEqual({ + status: "error", + message: "Use a session ID returned by find_sessions or start_session.", + retryable: false, + }) + expect(requests.some((request) => new URL(request.url).pathname.includes("ses_unknown"))).toBe(false) + + await execute("find_sessions", { + query: null, + scope: "current_project", + recency: "any", + limit: 10, + }) + expect(await execute("rename_session", { session_id: "ses_known", title: " " })).toEqual({ + status: "error", + message: "A non-empty session title is required.", + retryable: false, + }) + expect(await execute("rename_session", { session_id: "ses_known", title: "Old title" })).toEqual({ + status: "unchanged", + session_id: "ses_known", + previous_title: "Old title", + title: "Old title", + }) + expect(requests.some((request) => new URL(request.url).pathname.endsWith("/rename"))).toBe(false) + + expect(await execute("rename_session", { session_id: "ses_known", title: " New title " })).toEqual({ + status: "renamed", + session_id: "ses_known", + previous_title: "Old title", + title: "New title", + }) + const rename = requests.find((request) => new URL(request.url).pathname.endsWith("/rename")) + expect(rename?.method).toBe("POST") + expect(await rename?.json()).toEqual({ title: "New title" }) + } finally { + await bridge.close() + } +}) + +test("retries a lost completion wait and prompts a started session by ID", async () => { + const requests: Request[] = [] + const prompts: Array<{ readonly id: string; readonly text: string }> = [] + const notifications: Array = [] + const traces: Array<{ readonly event: string; readonly data?: Record }> = [] + let waits = 0 + const fetch = Object.assign( + async (input: string | URL | Request, init?: RequestInit) => { + const request = input instanceof Request ? input : new Request(input.toString(), init) + requests.push(request) + const url = new URL(request.url) + if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } }) + if (url.pathname === "/api/session" && request.method === "POST") + return Response.json({ + data: { + id: "ses_focused", + projectID: "project-1", + cost: "0", + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 1 }, + title: "Focused session", + location: { directory: "/workspace" }, + }, + }) + if (url.pathname === "/api/session/ses_focused/prompt" && request.method === "POST") { + const body = await request.json() + if ( + typeof body !== "object" || + body === null || + !("id" in body) || + typeof body.id !== "string" || + !("text" in body) || + typeof body.text !== "string" + ) + return new Response("Invalid prompt", { status: 400 }) + prompts.push({ id: body.id, text: body.text }) + return Response.json({ data: { id: body.id, admittedSeq: prompts.length } }) + } + if (url.pathname === "/api/session/ses_focused/wait" && request.method === "POST") { + waits += 1 + if (waits === 1) throw new Error("connection closed") + return new Response(null, { status: 204 }) + } + if (url.pathname === "/api/session/ses_focused/message" && request.method === "GET") + return Response.json({ + data: prompts.toReversed().map((prompt) => ({ id: prompt.id, type: "user", text: prompt.text })), + cursor: {}, + }) + return new Response("Not found", { status: 404 }) + }, + { preconnect: () => {} }, + ) + const bridge = await createOpenCodeBridge({ + client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }), + directory: "/workspace", + model: { providerID: "openai", id: "gpt-test" }, + notify: (notification) => notifications.push(notification), + trace: (event, data) => traces.push({ event, data }), + completionStore: memoryCompletionStore(), + }) + const execute = async (name: string, input: Record) => (await bridge.execute(name, input)).output + + try { + expect(bridge.definitions.some((tool) => tool.name === "continue_session")).toBe(false) + const started = await bridge.execute("start_session", { text: "First task", project_id: null }) + expect(started.output).toMatchObject({ + status: "started", + session_id: "ses_focused", + }) + expect(started.admittedPrompt).toEqual({ sessionID: "ses_focused", promptID: prompts[0].id }) + expect( + await execute("prompt_session", { session_id: "ses_focused", text: "Follow-up task" }), + ).toMatchObject({ + status: "started", + session_id: "ses_focused", + }) + + await Bun.sleep(1_100) + expect(prompts.map((prompt) => prompt.text)).toEqual(["First task", "Follow-up task"]) + expect(requests.filter((request) => new URL(request.url).pathname.endsWith("/wait")).length).toBeGreaterThanOrEqual( + 3, + ) + expect(traces.some((entry) => entry.event === "opencode.wait.retrying")).toBe(true) + expect(notifications.some((value) => JSON.stringify(value).includes("opencode.prompt.failed"))).toBe(false) + } finally { + await bridge.close() + } +}) + +test("discards a restored prompt whose session no longer exists", async () => { + const removed: Array<{ readonly sessionID: string; readonly promptID: string }> = [] + const handle = { sessionID: "ses_missing", promptID: "msg_orphaned" } + const fetch = Object.assign( + async (input: string | URL | Request) => { + const request = input instanceof Request ? input : new Request(input.toString()) + const url = new URL(request.url) + if (url.pathname === "/api/event") return new Response("", { headers: { "content-type": "text/event-stream" } }) + if (url.pathname === "/api/session/ses_missing") + return Response.json( + { _tag: "SessionNotFoundError", sessionID: handle.sessionID, message: `Session not found: ${handle.sessionID}` }, + { status: 404 }, + ) + return new Response("Not found", { status: 404 }) + }, + { preconnect: () => {} }, + ) + const store = { + ...memoryCompletionStore(), + entries: () => [{ status: "pending" as const, handle }], + remove: async (entry) => { + removed.push(entry) + }, + } satisfies CompletionStore + + const bridge = await createOpenCodeBridge({ + client: OpenCode.make({ baseUrl: "https://opencode.test", fetch }), + directory: "/workspace", + model: { providerID: "openai", id: "gpt-test" }, + notify: () => {}, + completionStore: store, + }) + try { + expect(removed).toEqual([handle]) + } finally { + await bridge.close() + } +}) + +function memoryCompletionStore(): CompletionStore { + return { + entries: () => [], + admitting: async () => {}, + pending: async () => {}, + completed: async () => {}, + remove: async () => {}, + close: async () => {}, + } +} diff --git a/packages/voice/test/pcm.test.ts b/packages/voice/test/pcm.test.ts new file mode 100644 index 0000000000..e3144d47bd --- /dev/null +++ b/packages/voice/test/pcm.test.ts @@ -0,0 +1,10 @@ +import { expect, test } from "bun:test" +import { pcmLevel } from "../src/pcm" + +test("PCM level ignores silence and normalizes audible energy", () => { + expect(pcmLevel(Buffer.alloc(4_800))).toBe(0) + const audible = Buffer.alloc(4_800) + for (let offset = 0; offset < audible.length; offset += 2) audible.writeInt16LE(8_000, offset) + expect(pcmLevel(audible)).toBeGreaterThan(0.8) + expect(pcmLevel(audible)).toBeLessThanOrEqual(1) +}) diff --git a/packages/voice/test/protocol.test.ts b/packages/voice/test/protocol.test.ts new file mode 100644 index 0000000000..3f7d939091 --- /dev/null +++ b/packages/voice/test/protocol.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, test } from "bun:test" +import { Option } from "effect" +import { decodeVoiceToolInput } from "../src/protocol" +import { createLiveContextAppendQueue, liveNotificationAcknowledgement } from "../src/protocol-live" +import { realtimeNotificationAcknowledgement, realtimeSessionUpdate } from "../src/protocol-realtime" +import { createSingleFlightAcknowledgement } from "../src/single-flight-acknowledgement" +import { openCodeAnnouncementText } from "../src/opencode-notification" + +describe("voice protocol", () => { + test("accepts only JSON object tool arguments", () => { + expect(Option.getOrUndefined(decodeVoiceToolInput('{"text":"hello","nested":{"value":1}}'))).toEqual({ + text: "hello", + nested: { value: 1 }, + }) + for (const input of ["not json", "null", "[]", "1", '"text"']) + expect(Option.getOrUndefined(decodeVoiceToolInput(input))).toBeUndefined() + }) + + test("correlates Realtime notification acknowledgements", () => { + const pending = { itemID: "item-current", eventID: "event-current" } + expect( + realtimeNotificationAcknowledgement( + { type: "conversation.item.created", item: { id: "item-current" } }, + pending, + ), + ).toBe(true) + expect( + realtimeNotificationAcknowledgement( + { type: "conversation.item.created", item: { id: "item-other" } }, + pending, + ), + ).toBeUndefined() + expect( + realtimeNotificationAcknowledgement( + { type: "error", error: { event_id: "event-current" } }, + pending, + ), + ).toBe(false) + }) + + test("preserves tool schemas in Realtime session updates", () => { + const tools = [ + { + type: "function" as const, + name: "open_form", + description: "Open form", + parameters: { + type: "object", + properties: { answer: { type: "object", additionalProperties: true } }, + additionalProperties: true, + }, + }, + ] + expect(JSON.parse(JSON.stringify(realtimeSessionUpdate({ instructions: "test", tools, voice: "marin" }))).session.tools) + .toEqual(tools) + }) + + test("correlates Live notification acknowledgements by expected context", () => { + const pending = { eventID: "event-current", acknowledgement: "delegation.context.appended" as const } + expect( + liveNotificationAcknowledgement( + { type: "delegation.context.appended", eventID: "event-current" }, + pending, + ), + ).toBe(true) + expect( + liveNotificationAcknowledgement({ type: "session.context.appended", eventID: "event-current" }, pending), + ).toBeUndefined() + expect( + liveNotificationAcknowledgement( + { type: "delegation.context.appended", eventID: "event-other" }, + pending, + ), + ).toBe(true) + expect( + liveNotificationAcknowledgement({ type: "error", errorEventID: "event-current" }, pending), + ).toBe(false) + }) + + test("serializes indistinguishable Live context appends", async () => { + const sent: Array> = [] + const queue = createLiveContextAppendQueue({ + send: (event) => { + sent.push(event) + return true + }, + timeoutMs: 100, + }) + const first = queue.append( + { type: "delegation.context.append", event_id: "event-1" }, + "delegation.context.appended", + ) + const second = queue.append( + { type: "delegation.context.append", event_id: "event-2" }, + "delegation.context.appended", + ) + expect(sent.map((event) => event["event_id"])).toEqual(["event-1"]) + + queue.receive({ type: "session.context.appended", eventID: "unrelated" }) + expect(sent.map((event) => event["event_id"])).toEqual(["event-1"]) + queue.receive({ type: "delegation.context.appended", eventID: "not-echoed" }) + expect(await first).toBe(true) + expect(sent.map((event) => event["event_id"])).toEqual(["event-1", "event-2"]) + queue.receive({ type: "delegation.context.appended" }) + expect(await second).toBe(true) + queue.close() + }) + + test("keeps notification acknowledgement single-flight", async () => { + const acknowledgement = createSingleFlightAcknowledgement<{ readonly token: string }>(50) + const first = acknowledgement.begin("notification-1", { token: "first" }) + const second = acknowledgement.begin("notification-2", { token: "second" }) + + expect(first.started).toBe(true) + expect(second.started).toBe(false) + expect(await second.promise).toBe(false) + expect(acknowledgement.current()).toEqual({ id: "notification-1", correlation: { token: "first" } }) + acknowledgement.settle("notification-1", true) + expect(await first.promise).toBe(true) + expect(acknowledgement.current()).toBeUndefined() + }) + + test("renders bounded conversational OpenCode announcements without identifiers", () => { + const text = openCodeAnnouncementText({ + type: "opencode.prompt.completed", + session_id: "ses_private", + prompt_id: "msg_private", + status: "completed", + text: "finished ".repeat(200), + }) + expect(text.length).toBeLessThanOrEqual(800) + expect(text).not.toContain("ses_private") + expect(text).not.toContain("msg_private") + expect(text).toStartWith("OpenCode prompt completed: finished") + }) + +}) diff --git a/packages/voice/test/realtime-projector.test.ts b/packages/voice/test/realtime-projector.test.ts new file mode 100644 index 0000000000..05c4afbc4d --- /dev/null +++ b/packages/voice/test/realtime-projector.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "bun:test" +import { createRealtimeEventProjector } from "../src/protocol-realtime" + +test("projects one tool start and one work request for duplicate item frames", () => { + const projector = createRealtimeEventProjector() + const item = { + type: "function_call", + name: "find_sessions", + call_id: "call-1", + arguments: '{"query":null}', + } + const added = projector.receive(JSON.stringify({ type: "response.output_item.added", item })) + const duplicate = projector.receive(JSON.stringify({ type: "response.output_item.added", item })) + const done = projector.receive(JSON.stringify({ type: "response.output_item.done", item })) + + expect(added.events).toContainEqual({ type: "tool.started", id: "call-1", name: "find_sessions" }) + expect(duplicate.events.some((event) => event.type === "tool.started")).toBe(false) + expect(done.events).toContainEqual({ + type: "work.requested", + request: { id: "call-1", name: "find_sessions", input: { query: null } }, + }) + expect(projector.receive(JSON.stringify({ type: "response.output_item.done", item })).events).toEqual([ + { type: "debug", message: "response.output_item.done" }, + ]) +}) + +test("resumes a response only after all projected work resolves", () => { + const projector = createRealtimeEventProjector() + const response = projector.receive( + JSON.stringify({ + type: "response.done", + response: { output: [{ type: "function_call", call_id: "call-1" }] }, + }), + ) + + expect(response.events).toContainEqual({ type: "assistant.done", awaitingWork: true }) + expect(response.commands).toEqual([]) + expect(projector.resolveWork("call-1")).toEqual([{ type: "response.create" }]) + expect(projector.resolveWork("call-1")).toEqual([]) +}) + +test("resumes when work resolved before the response completion frame", () => { + const projector = createRealtimeEventProjector() + expect(projector.resolveWork("call-1")).toEqual([]) + expect( + projector.receive( + JSON.stringify({ + type: "response.done", + response: { output: [{ type: "function_call", call_id: "call-1" }] }, + }), + ).commands, + ).toEqual([{ type: "response.create" }]) +}) + +test("rejects malformed tool arguments without hanging the provider call", () => { + const projector = createRealtimeEventProjector() + const result = projector.receive( + JSON.stringify({ + type: "response.output_item.done", + item: { type: "function_call", name: "find_sessions", call_id: "call-1", arguments: "[]" }, + }), + ) + + expect(result.events).toContainEqual({ + type: "work.rejected", + request: { id: "call-1", name: "find_sessions", input: {} }, + output: { status: "error", message: "Invalid arguments for tool find_sessions." }, + }) +}) diff --git a/packages/voice/test/responses-controller.test.ts b/packages/voice/test/responses-controller.test.ts new file mode 100644 index 0000000000..cffd7852a5 --- /dev/null +++ b/packages/voice/test/responses-controller.test.ts @@ -0,0 +1,229 @@ +import { expect, test } from "bun:test" +import { + createResponsesControllerContext, + responsesControllerTools, + runResponsesController, +} from "../src/responses-controller" + +test("runs the client delegation controller through OpenCode tools", async () => { + const bodies: Array> = [] + const calls: Array = [] + const responses = [ + { + id: "response-1", + output: [ + { + type: "function_call", + call_id: "call-1", + name: "start_session", + arguments: '{"text":"Inspect the package","project_id":null}', + }, + ], + }, + { + id: "response-2", + output: [ + { + type: "message", + content: [{ type: "output_text", text: "The inspection is running and I will report back." }], + }, + ], + }, + ] + const fetch = Object.assign( + async (_input: string | URL | Request, init?: RequestInit) => { + const body: unknown = await new Request("https://opencode.test", init).json() + if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body))) + return Response.json(responses.shift()) + }, + { preconnect: () => {} }, + ) + + const text = await runResponsesController({ + apiKey: "test", + model: "gpt-test", + instructions: "Use tools.", + text: "Inspect this package", + tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }], + execute: async (call) => { + calls.push(call) + return { status: "started", session_id: "ses_started", prompt_id: "prompt-1" } + }, + fetch, + }) + + expect(text).toBe( + "The inspection is running and I will report back.\n\nPrivate voice-control context (never speak this aloud): OpenCode session IDs explicitly used by this delegation: ses_started. Include the relevant session_id in any future delegated request to continue this work.", + ) + expect(calls).toEqual([ + { + id: "call-1", + name: "start_session", + input: { text: "Inspect the package", project_id: null }, + }, + ]) + expect(bodies[1]).toMatchObject({ + previous_response_id: "response-1", + input: [ + { + type: "function_call_output", + call_id: "call-1", + output: '{"status":"started","session_id":"ses_started","prompt_id":"prompt-1"}', + }, + ], + }) + expect(bodies[0]?.["tools"]).toEqual([ + { type: "function", name: "start_session", description: "Start", parameters: {} }, + ]) +}) + +test("rejects malformed controller tool arguments", async () => { + const fetch = Object.assign( + async () => + Response.json({ + id: "response-1", + output: [{ type: "function_call", call_id: "call-1", name: "start_session", arguments: "not json" }], + }), + { preconnect: () => {} }, + ) + + expect( + runResponsesController({ + apiKey: "test", + model: "gpt-test", + instructions: "Use tools.", + text: "Inspect this package", + tools: [{ type: "function", name: "start_session", description: "Start", parameters: {} }], + execute: async () => ({ status: "started" }), + fetch, + }), + ).rejects.toThrow("invalid arguments for start_session") +}) + +test("retains the most recently used session across delegations", async () => { + const context = createResponsesControllerContext() + const bodies: Array> = [] + const responses = [ + { + id: "response-1", + output: [ + { + type: "function_call", + call_id: "call-1", + name: "read_session", + arguments: '{"session_id":"ses_recent"}', + }, + ], + }, + { + id: "response-2", + output: [{ type: "message", content: [{ type: "output_text", text: "Found it." }] }], + }, + { + id: "response-3", + output: [{ type: "message", content: [{ type: "output_text", text: "Stopped it." }] }], + }, + ] + const fetch = Object.assign( + async (_input: string | URL | Request, init?: RequestInit) => { + const body: unknown = await new Request("https://opencode.test", init).json() + if (body && typeof body === "object" && !Array.isArray(body)) bodies.push(Object.fromEntries(Object.entries(body))) + return Response.json(responses.shift()) + }, + { preconnect: () => {} }, + ) + const options = { + apiKey: "test", + model: "gpt-test", + instructions: "Use tools.", + tools: [{ type: "function" as const, name: "read_session", description: "Read", parameters: {} }], + execute: async () => ({ status: "ok" }), + context, + fetch, + } + + await runResponsesController({ ...options, text: "Find the label behavior session" }) + await runResponsesController({ ...options, text: "Stop that session" }) + + expect(context.lastSessionID).toBe("ses_recent") + expect(bodies[2]?.["input"]).toContain("the most recently used OpenCode session ID is ses_recent") + expect(bodies[2]?.["input"]).toContain('references like "that session", "it", or "stop that"') +}) + +test("coalesces identical read-only calls within one delegation", async () => { + const responses = [ + { + id: "response-1", + output: [ + { type: "function_call", call_id: "call-1", name: "read_session", arguments: '{"session_id":"ses_1"}' }, + { type: "function_call", call_id: "call-2", name: "read_session", arguments: '{"session_id":"ses_1"}' }, + ], + }, + { + id: "response-2", + output: [{ type: "message", content: [{ type: "output_text", text: "Done." }] }], + }, + ] + let executions = 0 + const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} }) + + await runResponsesController({ + apiKey: "test", + model: "gpt-test", + instructions: "Use tools.", + text: "Read it", + tools: [{ type: "function", name: "read_session", description: "Read", parameters: {} }], + execute: async () => { + executions += 1 + return { status: "ok" } + }, + fetch, + }) + + expect(executions).toBe(1) +}) + +test("withholds new-session creation during follow-ups unless explicitly requested", () => { + const context = createResponsesControllerContext() + context.lastSessionID = "ses_recent" + const tools = [ + { type: "function" as const, name: "start_session", description: "Start", parameters: {} }, + { type: "function" as const, name: "read_session", description: "Read", parameters: {} }, + ] + + expect(responsesControllerTools(tools, "Stop that session", context).map((tool) => tool.name)).toEqual([ + "read_session", + ]) + expect(responsesControllerTools(tools, "Start a new session for this", context)).toEqual(tools) +}) + +test("remembers a uniquely discovered session without requiring a read", async () => { + const context = createResponsesControllerContext() + const responses = [ + { + id: "response-1", + output: [ + { type: "function_call", call_id: "call-1", name: "find_sessions", arguments: '{"query":"label"}' }, + ], + }, + { + id: "response-2", + output: [{ type: "message", content: [{ type: "output_text", text: "Found one." }] }], + }, + ] + const fetch = Object.assign(async () => Response.json(responses.shift()), { preconnect: () => {} }) + + await runResponsesController({ + apiKey: "test", + model: "gpt-test", + instructions: "Use tools.", + text: "Find label behavior", + tools: [{ type: "function", name: "find_sessions", description: "Find", parameters: {} }], + execute: async () => ({ status: "ok", sessions: [{ id: "ses_unique", title: "Label behavior" }] }), + context, + fetch, + }) + + expect(context.lastSessionID).toBe("ses_unique") + expect(context.sessionIDs).toEqual(new Set(["ses_unique"])) +}) diff --git a/packages/voice/test/ui-layout.test.ts b/packages/voice/test/ui-layout.test.ts new file mode 100644 index 0000000000..65b9367396 --- /dev/null +++ b/packages/voice/test/ui-layout.test.ts @@ -0,0 +1,331 @@ +import { expect, mock, test } from "bun:test" +import { createTestRenderer } from "@opentui/core/testing" + +test("voice rows fill the viewport, wrap under content, and keep tools after their preamble", async () => { + const setup = await createTestRenderer({ width: 64, height: 18, useThread: false }) + const core = await import("@opentui/core") + void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer })) + const { createVoiceTUI } = await import("../src/ui") + let interrupts = 0 + let voiceCycles = 0 + let microphoneToggles = 0 + let speakerToggles = 0 + const ui = await createVoiceTUI({ + onInterrupt: () => interrupts++, + onExit: () => {}, + onCycleVoice: () => voiceCycles++, + onToggleMicrophone: () => microphoneToggles++, + onToggleSpeaker: () => speakerToggles++, + reducedMotion: true, + }) + + try { + ui.setStatus({ audio: "duplex", voice: "marin", model: "gpt-live" }) + ui.meta("[session] created session-1") + ui.userCommitted("user-1") + ui.userTranscript("user-1", "Please inspect the current session", true) + ui.toolStart("tool-1", "opencode", { request: "inspect" }) + ui.meta("[session] adopted session-1") + ui.assistantDelta("Let me check. This sentence is") + ui.assistantDelta(" long enough to wrap cleanly under its column.") + ui.assistantDone() + ui.assistantDelta("Continued in the same assistant row.") + ui.assistantDone() + await Bun.sleep(200) + await setup.flush() + + const rows = frameRows(setup.captureCharFrame()) + expect(rows).toHaveLength(18) + expect(rows.every((row) => row.length === 64)).toBe(true) + expect(rows[16]).toStartWith(" ▂▂▂▁ duplex marin gpt-live") + expect(rows[16]).not.toContain("no session") + expect(rows[17]).toContain("esc interrupt") + expect(rows[17]).not.toContain("any key") + expect(rows[15].trim()).toBe("") + expect(rows.filter((row) => row.includes("Let me check"))).toHaveLength(1) + expect(rows.findIndex((row) => row.includes("opencode"))).toBeGreaterThan( + rows.findIndex((row) => row.includes("Let me check")), + ) + expect(rows.findIndex((row) => row.includes("[session] adopted"))).toBeGreaterThan( + rows.findIndex((row) => row.includes("opencode")), + ) + expect(rows.find((row) => row.includes("under its column"))?.indexOf("under")).toBe( + rows.find((row) => row.includes("Let me check"))?.indexOf("Let"), + ) + // Deltas inside one turn concatenate raw; a delta after assistantDone starts a new + // message and must gain exactly one space at the boundary. + const assistant = rows.join("") + expect(assistant).toContain("This sentence is long enough") + expect(assistant).toContain("column. Continued") + expect(assistant).not.toContain("column.Continued") + expect(rows.find((row) => row.includes("[session] created"))).toStartWith(" · [session]") + + setup.resize(96, 22) + await setup.flush() + const wide = frameRows(setup.captureCharFrame()) + expect(wide).toHaveLength(22) + expect(wide.every((row) => row.length === 96)).toBe(true) + expect(wide.at(-2)).toStartWith(" ▂▂▂▁ duplex marin gpt-live") + + setup.resize(48, 16) + await setup.flush() + const narrow = frameRows(setup.captureCharFrame()) + expect(narrow).toHaveLength(16) + expect(narrow.every((row) => row.length === 48)).toBe(true) + expect(narrow.at(-1)).toContain("esc interrupt") + + setup.resize(64, 18) + await setup.flush() + + ui.userSpeaking(true) + await setup.flush() + expect(setup.captureCharFrame()).not.toContain("│ transcribing") + expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ▂▂▂▂") + + ui.userReset() + await setup.flush() + expect(setup.captureCharFrame()).not.toContain("│ transcribing") + + ui.userSpeaking(true) + ui.userSpeaking(false) + ui.userCommitted("user-2") + ui.userTranscript("user-2", "partial transcript", false) + await setup.flush() + const partial = setup + .captureCharFrame() + .split("\n") + .find((row) => row.includes("partial transcript")) + expect(partial).toContain("│ partial transcript") + expect(partial).not.toContain("you") + + ui.userSpeaking(true) + await setup.flush() + expect( + setup + .captureCharFrame() + .split("\n") + .filter((row) => row.includes("│ transcribing")), + ).toHaveLength(0) + + ui.userReset() + await setup.flush() + + ui.setStatus({ microphoneMuted: true }) + await setup.flush() + expect(frameRows(setup.captureCharFrame()).at(-2)).toStartWith(" ────") + const muted = setup + .captureCharFrame() + .split("\n") + .find((row) => row.includes("partial transcript")) + expect(muted).toContain("│ partial transcript") + expect(muted).not.toContain("│ ...") + + ui.userSpeaking(false) + ui.setStatus({ microphoneMuted: false }) + ui.userTranscript("user-2", "partial transcript", true) + await setup.flush() + const complete = setup + .captureCharFrame() + .split("\n") + .find((row) => row.includes("partial transcript")) + expect(complete).toContain("│ partial transcript") + + ui.assistantDelta("First response.") + ui.userCommitted("boundary-user") + ui.userTranscript("boundary-user", "A new request", true) + ui.assistantDelta("Second response.") + await setup.flush() + const boundaries = setup.captureCharFrame().split("\n") + expect(boundaries.findIndex((row) => row.includes("First response."))).toBeLessThan( + boundaries.findIndex((row) => row.includes("A new request")), + ) + expect(boundaries.findIndex((row) => row.includes("A new request"))).toBeLessThan( + boundaries.findIndex((row) => row.includes("Second response.")), + ) + + await setup.mockInput.typeText(" x") + setup.mockInput.pressArrow("right") + await setup.flush() + expect(interrupts).toBe(0) + expect(voiceCycles).toBe(0) + expect(microphoneToggles).toBe(0) + expect(speakerToggles).toBe(0) + + setup.mockInput.pressKey("v") + setup.mockInput.pressKey("m") + setup.mockInput.pressKey("s") + await setup.flush() + expect(voiceCycles).toBe(1) + expect(microphoneToggles).toBe(1) + expect(speakerToggles).toBe(1) + + setup.mockInput.pressEscape() + await Bun.sleep(50) + await setup.flush() + expect(interrupts).toBe(1) + } finally { + ui.close() + mock.restore() + } +}) + +// In audio mode ui.assistantDone() is scheduled off the AudioSession playback clock, +// so the next turn's transcript deltas arrive while the row is still streaming. The turn boundary +// therefore has to survive in the delta text itself, not in the row's streaming flag. +test("keeps the turn boundary when the next turn streams before playback finishes", async () => { + const setup = await createTestRenderer({ width: 72, height: 12, useThread: false }) + const core = await import("@opentui/core") + void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer })) + const { createVoiceTUI } = await import("../src/ui") + const ui = await createVoiceTUI({ + onInterrupt: () => {}, + onExit: () => {}, + onCycleVoice: () => {}, + onToggleMicrophone: () => {}, + onToggleSpeaker: () => {}, + reducedMotion: true, + }) + + try { + // Verbatim projector deltas: each turn's first fragment carries the boundary space. + ui.assistantDelta(' Session "Fix auth bug"') + ui.assistantDelta(" is ready.") + // turn.done reaches the protocol, but playback is still draining, so no assistantDone() yet. + ui.assistantDelta(" Next up, tests.") + await setup.flush() + + const frame = setup.captureCharFrame() + expect(frame).toContain('Session "Fix auth bug" is ready. Next up, tests.') + expect(frame).not.toContain("ready.Next") + // The row itself must not open with the boundary space. + expect(frame).toContain('│ Session "Fix auth bug"') + expect(frame).not.toContain('│ Session "Fix auth bug"') + + ui.assistantDone() + await setup.flush() + expect(setup.captureCharFrame()).toContain("is ready. Next up, tests.") + } finally { + ui.close() + mock.restore() + } +}) + +test("keeps animated text mounted after its reveal completes", async () => { + const setup = await createTestRenderer({ width: 72, height: 10, useThread: false }) + const core = await import("@opentui/core") + void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer })) + const { createVoiceTUI } = await import("../src/ui") + const ui = await createVoiceTUI({ + onInterrupt: () => {}, + onExit: () => {}, + onCycleVoice: () => {}, + onToggleMicrophone: () => {}, + onToggleSpeaker: () => {}, + }) + + try { + ui.assistantDelta("Animated text remains visible.") + await Bun.sleep(500) + await setup.flush() + expect(setup.captureCharFrame()).toContain("Animated text remains visible.") + } finally { + ui.close() + mock.restore() + } +}) + +test("renders consecutive tools as compact single-spaced rows", async () => { + const setup = await createTestRenderer({ width: 52, height: 8, useThread: false }) + const core = await import("@opentui/core") + void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer })) + const { createVoiceTUI } = await import("../src/ui") + const ui = await createVoiceTUI({ + onInterrupt: () => {}, + onExit: () => {}, + onCycleVoice: () => {}, + onToggleMicrophone: () => {}, + onToggleSpeaker: () => {}, + reducedMotion: true, + }) + + try { + ui.toolStart("tool-1", "read_session", {}) + ui.toolDone("tool-1", { status: "ok" }) + ui.toolStart("tool-2", "read_session", {}) + ui.toolDone("tool-2", { status: "ok" }) + await Bun.sleep(200) + await setup.flush() + + const rows = frameRows(setup.captureCharFrame()) + const tools = rows.flatMap((row, index) => (row.includes("read_session") ? [{ row, index }] : [])) + expect(tools).toHaveLength(2) + expect(tools[1]?.index).toBe((tools[0]?.index ?? 0) + 1) + expect(tools.every((tool) => tool.row.includes("✓ read_session ok"))).toBe(true) + expect(rows.at(-3)?.trim()).toBe("") + } finally { + ui.close() + mock.restore() + } +}) + +test("shows the reason a tool failed", async () => { + const setup = await createTestRenderer({ width: 72, height: 8, useThread: false }) + const core = await import("@opentui/core") + void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer })) + const { createVoiceTUI } = await import("../src/ui") + const ui = await createVoiceTUI({ + onInterrupt: () => {}, + onExit: () => {}, + onCycleVoice: () => {}, + onToggleMicrophone: () => {}, + onToggleSpeaker: () => {}, + reducedMotion: true, + }) + + try { + ui.toolStart("tool-error", "opencode", {}) + ui.toolDone("tool-error", { status: "error", message: "Invalid tool schema" }) + await Bun.sleep(200) + await setup.flush() + expect(setup.captureCharFrame()).toContain("opencode error · Invalid tool schema") + } finally { + ui.close() + mock.restore() + } +}) + +test("paints the full viewport and keeps mic activity in the footer", async () => { + const setup = await createTestRenderer({ width: 40, height: 12, useThread: false }) + const core = await import("@opentui/core") + void mock.module("@opentui/core", () => ({ ...core, createCliRenderer: async () => setup.renderer })) + const { createVoiceTUI } = await import("../src/ui") + const ui = await createVoiceTUI({ + onInterrupt: () => {}, + onExit: () => {}, + onCycleVoice: () => {}, + onToggleMicrophone: () => {}, + onToggleSpeaker: () => {}, + reducedMotion: true, + }) + + try { + ui.userSpeaking(true) + await setup.flush() + + const rows = frameRows(setup.captureCharFrame()) + expect(rows.some((row) => row.includes("│ transcribing"))).toBe(false) + expect(rows.at(-2)).toStartWith(" ▂▂▂▂") + expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true) + + setup.resize(56, 16) + await setup.flush() + expect(setup.captureSpans().lines.every((line) => line.spans.every((span) => span.bg.a === 1))).toBe(true) + } finally { + ui.close() + mock.restore() + } +}) + +function frameRows(frame: string) { + return frame.split("\n").slice(0, -1) +} diff --git a/packages/voice/test/ui-model.test.ts b/packages/voice/test/ui-model.test.ts new file mode 100644 index 0000000000..2464769c24 --- /dev/null +++ b/packages/voice/test/ui-model.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, test } from "bun:test" +import { initialVoiceView, transitionVoiceView, type VoiceViewEvent, type VoiceViewState } from "../src/ui-model" + +const apply = (state: VoiceViewState, ...events: ReadonlyArray) => + events.reduce(transitionVoiceView, state) + +describe("VoiceView", () => { + test("keeps assistant responses separated by a committed user turn", () => { + const state = apply( + initialVoiceView(), + { type: "assistant.delta", text: "First response.", now: 0, animate: false }, + { type: "user.committed", itemID: "user-1" }, + { type: "user.transcript", itemID: "user-1", text: "Next request", final: true, now: 1, animate: false }, + { type: "assistant.delta", text: "Second response.", now: 2, animate: false }, + ) + expect(state.messages.map((message) => message.kind)).toEqual(["assistant", "user", "assistant"]) + expect(state.messages.flatMap((message) => ("text" in message ? [message.text] : []))).toEqual([ + "First response.", + "Next request", + "Second response.", + ]) + }) + + test("keeps local sound activity out of transcript history", () => { + const started = transitionVoiceView(initialVoiceView(), { type: "user.started" }) + expect(started.messages).toEqual([]) + const committed = transitionVoiceView(started, { type: "user.committed", itemID: "user-1" }) + expect(committed.messages).toEqual([ + { key: "message-1", kind: "user", itemID: "user-1", transcribing: true, reveals: [] }, + ]) + + const abandoned = apply(initialVoiceView(), { type: "user.started" }, { type: "user.reset" }) + expect(abandoned.messages).toEqual([]) + }) + + test("deduplicates transcript snapshots and clears completed reveals", () => { + const state = apply( + initialVoiceView(), + { type: "user.committed", itemID: "user-1" }, + { type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 0, animate: true }, + { type: "user.transcript", itemID: "user-1", text: "Hello", final: false, now: 10, animate: true }, + ) + expect(state.messages).toHaveLength(1) + expect(state.messages[0]?.kind === "user" ? state.messages[0].reveals.length : 0).toBeGreaterThan(0) + expect(transitionVoiceView(state, { type: "reveals.completed" }).messages).toEqual([ + { key: "message-1", kind: "user", itemID: "user-1", text: "Hello", transcribing: true, reveals: [] }, + ]) + }) + + test("bounds message history", () => { + const state = Array.from({ length: 205 }, (_, index) => index).reduce( + (current, index) => transitionVoiceView(current, { type: "meta", text: String(index) }), + initialVoiceView(), + ) + expect(state.messages).toHaveLength(200) + expect(state.messages[0]).toEqual({ key: "message-6", kind: "meta", text: "5" }) + }) + + test("settles tool completion before or after insertion", () => { + const inserted = apply( + initialVoiceView(), + { type: "tool.started", callID: "call-1", name: "read_session", input: {} }, + { type: "tool.done", callID: "call-1", output: { status: "ok" } }, + ) + expect(inserted.messages[0]).toEqual({ + key: "message-1", + kind: "tool", + callID: "call-1", + name: "read_session", + input: {}, + output: { status: "ok" }, + }) + }) +}) diff --git a/packages/voice/test/voice-coordinator.test.ts b/packages/voice/test/voice-coordinator.test.ts new file mode 100644 index 0000000000..a8ee3d9680 --- /dev/null +++ b/packages/voice/test/voice-coordinator.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, test } from "bun:test" +import { initialVoiceState, transitionVoice, type VoiceEvent, type VoiceState } from "../src/voice-coordinator" + +const apply = (state: VoiceState, ...events: ReadonlyArray) => + events.reduce( + (result, event) => { + const next = transitionVoice(result.state, event) + return { state: next.state, commands: [...result.commands, ...next.commands] } + }, + { state, commands: [] as Array["commands"][number]> }, + ) + +describe("VoiceCoordinator", () => { + test("delivers one notification at a time when the server becomes idle", () => { + const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" } + const waiting = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "user.committed" }, + { type: "notification.queued", notification }, + ) + expect(waiting.commands).toEqual([]) + + expect(transitionVoice(waiting.state, { type: "assistant.done", awaitingWork: false })).toEqual({ + state: { + ...waiting.state, + conversation: "waiting", + assistant: "idle", + notifications: [], + delivery: { notification, phase: "sending" }, + }, + commands: [ + { type: "assistant.finish", awaitingWork: false, notificationID: undefined }, + { type: "notification.send", notification }, + ], + }) + }) + + test("keeps notifications queued while the user is speaking", () => { + const notification = { id: "notification-1", text: "completed" } + const result = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "user.started" }, + { type: "notification.queued", notification }, + ) + expect(result.commands).toEqual([]) + expect(result.state.notifications).toEqual([notification]) + expect(transitionVoice(result.state, { type: "user.stopped" }).commands).toEqual([ + { type: "notification.send", notification }, + ]) + }) + + test("keeps notifications queued until active tool calls finish", () => { + const notification = { id: "notification-1", text: "completed" } + const result = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "tool.started", id: "call-1" }, + { type: "assistant.done", awaitingWork: false }, + { type: "notification.queued", notification }, + ) + expect(result.commands).toEqual([ + { type: "assistant.finish", awaitingWork: false, notificationID: undefined }, + ]) + expect(transitionVoice(result.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([ + { type: "notification.send", notification }, + ]) + }) + + test("tracks tools by call ID and reconnects only after work and conversation settle", () => { + const busy = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "tool.started", id: "call-1" }, + { type: "voice.selected", voice: "cedar" }, + ) + expect(busy.commands).toEqual([]) + expect(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).commands).toEqual([]) + + const idle = transitionVoice(transitionVoice(busy.state, { type: "tool.finished", id: "call-1" }).state, { + type: "assistant.done", + awaitingWork: false, + }) + expect(idle.commands).toEqual([ + { type: "assistant.finish", awaitingWork: false, notificationID: undefined }, + { type: "connection.reconnect" }, + ]) + expect(idle.state.connection).toBe("connecting") + }) + + test("a user commit closes assistant suppression and establishes a waiting boundary", () => { + const suppressed = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "assistant.started" }, + { type: "assistant.suppressed" }, + { type: "user.committed" }, + ) + expect(suppressed.state.assistant).toBe("idle") + expect(suppressed.state.conversation).toBe("waiting") + }) + + test("requeues a notification when the adapter rejects it", () => { + const notification = { id: "notification-1", text: "completed" } + const sent = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "notification.queued", notification }, + ) + const failed = transitionVoice(sent.state, { type: "notification.failed", id: notification.id }) + expect(failed.state.notifications).toEqual([notification]) + expect(failed.state.connection).toBe("connecting") + expect(failed.commands).toEqual([{ type: "connection.reconnect" }]) + }) + + test("keeps a notification in flight until its playback completes", () => { + const notification = { id: "notification-1", promptID: "prompt-1", text: "completed" } + const sent = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { + type: "notification.queued", + notification, + }, + ) + expect(sent.state.delivery).toEqual({ notification, phase: "sending" }) + + const accepted = transitionVoice(sent.state, { type: "notification.accepted", id: notification.id }).state + expect(accepted.delivery).toEqual({ notification, phase: "accepted" }) + const speaking = transitionVoice(accepted, { type: "assistant.started" }).state + expect(speaking.delivery).toEqual({ notification, phase: "announcing" }) + const generated = transitionVoice(speaking, { type: "assistant.done", awaitingWork: false }) + expect(generated.state.delivery).toEqual({ notification, phase: "awaiting-playback" }) + expect(generated.commands).toEqual([ + { type: "assistant.finish", awaitingWork: false, notificationID: notification.id }, + ]) + expect(transitionVoice(generated.state, { type: "notification.announced", id: notification.id })).toEqual({ + state: { ...generated.state, delivery: undefined }, + commands: [{ type: "notification.delivered", notification }], + }) + }) + + test("does not attribute an already-active assistant turn to a newly appended notification", () => { + const notification = { id: "notification-1", text: "completed" } + const active = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "notification.queued", notification }, + { type: "assistant.started" }, + { type: "notification.accepted", id: notification.id }, + ).state + + expect(active.delivery).toEqual({ notification, phase: "accepted" }) + const existingDone = transitionVoice(active, { type: "assistant.done", awaitingWork: false }) + expect(existingDone.state.delivery).toEqual({ notification, phase: "accepted" }) + expect(existingDone.commands).toEqual([ + { type: "assistant.finish", awaitingWork: false, notificationID: undefined }, + ]) + expect(transitionVoice(existingDone.state, { type: "assistant.started" }).state.delivery).toEqual({ + notification, + phase: "announcing", + }) + }) + + test("keeps assistant activity independent from a full-duplex user commit", () => { + const active = apply(initialVoiceState("marin"), { type: "connection.ready" }, { type: "assistant.started" }).state + const committed = transitionVoice(active, { type: "user.committed" }).state + expect(committed.assistant).toBe("active") + expect(committed.conversation).toBe("responding") + }) + + test("ignores stale notification callbacks", () => { + const notification = { id: "notification-2", text: "current" } + const sent = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "notification.queued", notification }, + ).state + + expect(transitionVoice(sent, { type: "notification.accepted", id: "notification-1" })).toEqual({ + state: sent, + commands: [], + }) + expect(transitionVoice(sent, { type: "notification.failed", id: "notification-1" })).toEqual({ + state: sent, + commands: [], + }) + expect(transitionVoice(sent, { type: "notification.announced", id: "notification-1" })).toEqual({ + state: sent, + commands: [], + }) + }) + + test("waits for one announcement playback before sending the next", () => { + const first = { id: "notification-1", text: "first" } + const second = { id: "notification-2", text: "second" } + const sent = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "notification.queued", notification: first }, + { type: "notification.queued", notification: second }, + { type: "notification.accepted", id: first.id }, + { type: "assistant.started" }, + { type: "assistant.done", awaitingWork: false }, + ) + expect(sent.state.delivery).toEqual({ notification: first, phase: "awaiting-playback" }) + expect(sent.state.notifications).toEqual([second]) + + expect(transitionVoice(sent.state, { type: "notification.announced", id: first.id })).toEqual({ + state: { + ...sent.state, + conversation: "waiting", + delivery: { notification: second, phase: "sending" }, + notifications: [], + }, + commands: [ + { type: "notification.delivered", notification: first }, + { type: "notification.send", notification: second }, + ], + }) + }) + + test("keeps an announcement active across tool work", () => { + const notification = { id: "notification-1", text: "completed" } + const announcing = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "notification.queued", notification }, + { type: "notification.accepted", id: notification.id }, + { type: "assistant.started" }, + ).state + expect(transitionVoice(announcing, { type: "assistant.done", awaitingWork: true }).state.delivery).toEqual({ + notification, + phase: "announcing", + }) + }) + + test("requeues an interrupted announcement and reconnects before retrying", () => { + const notification = { id: "notification-1", text: "completed" } + const awaitingPlayback = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "notification.queued", notification }, + { type: "notification.accepted", id: notification.id }, + { type: "assistant.started" }, + { type: "assistant.done", awaitingWork: false }, + ).state + const interrupted = transitionVoice(awaitingPlayback, { + type: "notification.interrupted", + id: notification.id, + }) + expect(interrupted.state.connection).toBe("connecting") + expect(interrupted.state.notifications).toEqual([notification]) + expect(interrupted.commands).toEqual([{ type: "connection.reconnect" }]) + }) + + test("holds an interrupted announcement reconnect until barge-in speech stops", () => { + const notification = { id: "notification-1", text: "completed" } + const announcing = apply( + initialVoiceState("marin"), + { type: "connection.ready" }, + { type: "notification.queued", notification }, + { type: "notification.accepted", id: notification.id }, + { type: "assistant.started" }, + ).state + const barged = transitionVoice(announcing, { type: "user.started", bargeIn: true }) + expect(barged.state.notifications).toEqual([notification]) + expect(barged.state.userSpeaking).toBe(true) + expect(barged.state.connection).toBe("ready") + expect(barged.commands).toEqual([{ type: "assistant.interrupt" }]) + + expect(transitionVoice(barged.state, { type: "user.stopped" }).commands).toEqual([ + { type: "connection.reconnect" }, + ]) + }) +}) diff --git a/packages/voice/test/voice-session.test.ts b/packages/voice/test/voice-session.test.ts new file mode 100644 index 0000000000..68545a768c --- /dev/null +++ b/packages/voice/test/voice-session.test.ts @@ -0,0 +1,233 @@ +import { expect, mock, test } from "bun:test" +import type { AudioSession } from "../src/audio-session" +import type { VoiceConnection, VoiceProtocol, VoiceProtocolEvent, VoiceProtocolOptions } from "../src/protocol" +import type { VoiceUI } from "../src/ui" +import { createVoiceSession } from "../src/voice-session" + +test("ignores events from a replaced connection", async () => { + const setup = makeSession() + setup.session.start() + setup.connections[0].emit({ type: "ready" }) + setup.session.cycleVoice() + await Bun.sleep(0) + + expect(setup.connections).toHaveLength(2) + setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "stale" }) + setup.connections[1].emit({ type: "assistant.transcript.delta", delta: "current" }) + expect(setup.assistantText).toEqual(["current"]) +}) + +test("resolves work against the connection that requested it", async () => { + const setup = makeSession() + setup.session.start() + setup.connections[0].emit({ type: "ready" }) + setup.connections[0].emit({ type: "tool.started", id: "call-1", name: "find_sessions" }) + setup.connections[0].emit({ + type: "work.requested", + request: { id: "call-1", name: "find_sessions", input: { query: null } }, + }) + await Bun.sleep(0) + + expect(setup.resolvedWork).toEqual([ + { + request: { id: "call-1", name: "find_sessions", input: { query: null } }, + output: { status: "ok" }, + }, + ]) +}) + +test("acknowledges a durable notification only after playback", async () => { + const setup = makeSession() + setup.session.start() + setup.connections[0].emit({ type: "ready" }) + setup.session.queueNotification({ + notification: { + type: "opencode.prompt.completed", + session_id: "session-1", + prompt_id: "prompt-1", + status: "completed", + text: "done", + }, + receipt: { sessionID: "session-1", promptID: "prompt-1" }, + }) + await Bun.sleep(0) + + setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." }) + setup.connections[0].emit({ type: "assistant.done", awaitingWork: false }) + expect(setup.delivered).toEqual([]) + + setup.finishPlayback?.("played") + await Bun.sleep(0) + expect(setup.delivered).toEqual(["prompt-1"]) +}) + +test("retries a notification whose playback was interrupted", async () => { + const setup = makeSession() + setup.session.start() + setup.connections[0].emit({ type: "ready" }) + setup.session.queueNotification({ + notification: { + type: "opencode.prompt.completed", + session_id: "session-1", + prompt_id: "prompt-1", + status: "completed", + text: "done", + }, + receipt: { sessionID: "session-1", promptID: "prompt-1" }, + }) + await Bun.sleep(0) + setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Finished." }) + setup.connections[0].emit({ type: "assistant.done", awaitingWork: false }) + setup.finishPlayback?.("interrupted") + await Bun.sleep(0) + + expect(setup.delivered).toEqual([]) + expect(setup.connections).toHaveLength(2) + setup.connections[1].emit({ type: "ready" }) + await Bun.sleep(0) + expect(setup.connections[1].notifyCalls()).toBe(1) +}) + +test("closes owned resources once", async () => { + const setup = makeSession() + setup.session.start() + await Promise.all([setup.session.close(), setup.session.close()]) + expect(setup.audioClose).toHaveBeenCalledTimes(1) + expect(setup.toolClose).toHaveBeenCalledTimes(1) + expect(setup.connections[0].closeCalls()).toBe(1) +}) + +test("does not acknowledge nonterminal prompt notifications", async () => { + const setup = makeSession() + setup.session.start() + setup.connections[0].emit({ type: "ready" }) + setup.session.queueNotification({ + notification: { + type: "opencode.prompt.blocked", + prompt_id: "prompt-1", + blocker: "permission", + session_id: "session-1", + request_id: "permission-1", + action: "read", + resources: ["file.ts"], + }, + }) + await Bun.sleep(0) + setup.connections[0].emit({ type: "assistant.transcript.delta", delta: "Permission needed." }) + setup.connections[0].emit({ type: "assistant.done", awaitingWork: false }) + setup.finishPlayback?.("played") + await Bun.sleep(0) + + expect(setup.delivered).toEqual([]) +}) + +function makeSession() { + const connections: Array< + VoiceConnection & { emit(event: VoiceProtocolEvent): void; notifyCalls(): number; closeCalls(): number } + > = [] + const assistantText: string[] = [] + const resolvedWork: Array = [] + const delivered: string[] = [] + const audioClose = mock(() => {}) + const toolClose = mock(async () => {}) + let finishPlayback: ((outcome: "played" | "interrupted" | "inaudible") => void) | undefined + const ui = { + meta: mock(() => {}), + userSpeaking: mock(() => {}), + userReset: mock(() => {}), + userAudioLevel: mock(() => {}), + userCommitted: mock(() => {}), + userTranscript: mock(() => {}), + assistantAudio: mock(() => {}), + assistantPlaybackStopped: mock(() => {}), + assistantDelta: (text: string) => assistantText.push(text), + assistantTranscript: mock(() => {}), + assistantDone: mock(() => {}), + toolStart: mock(() => {}), + toolDone: mock(() => {}), + setStatus: mock(() => {}), + close: mock(() => {}), + } satisfies VoiceUI + const audio = { + fullDuplex: false, + microphoneMuted: false, + speakerMuted: false, + start: mock(async () => {}), + toggleMicrophone: mock(() => false), + toggleSpeaker: mock(() => false), + isPlaying: () => false, + play: mock(() => {}), + finishPlayback: () => { + const result = Promise.withResolvers<"played" | "interrupted" | "inaudible">() + finishPlayback = result.resolve + return result.promise + }, + flushPlayback: mock(() => {}), + noteUserCommitted: mock(() => {}), + noteUserTranscript: mock(() => {}), + close: audioClose, + } satisfies AudioSession + const protocol = { + name: "realtime", + inputActivity: "server", + capabilities: { textInput: true, interruption: true, delegation: false }, + connect(options: VoiceProtocolOptions) { + const notify = mock(async () => true) + const close = mock(async () => {}) + const connection = { + appendAudio: mock(() => {}), + sendText: mock(() => {}), + resolveWork: (request, output) => resolvedWork.push({ request, output }), + resolveDelegation: mock(() => {}), + notify, + interrupt: () => true, + close, + emit: options.onEvent, + notifyCalls: () => notify.mock.calls.length, + closeCalls: () => close.mock.calls.length, + } satisfies VoiceConnection & { + emit(event: VoiceProtocolEvent): void + notifyCalls(): number + closeCalls(): number + } + connections.push(connection) + return connection + }, + } satisfies VoiceProtocol + const session = createVoiceSession({ + protocol, + connection: { + apiKey: "test", + model: "voice-test", + instructions: "test", + tools: [], + fullDuplex: false, + debug: false, + }, + initialVoice: "marin", + voices: ["marin", "cedar"], + ui, + audio, + tools: { + execute: async () => ({ output: { status: "ok" } }), + acknowledge: async (receipt) => { + delivered.push(receipt.promptID) + }, + close: toolClose, + }, + delegate: async () => "done", + onClosed: () => {}, + }) + return { + session, + connections, + assistantText, + resolvedWork, + delivered, + audioClose, + toolClose, + get finishPlayback() { + return finishPlayback + }, + } +} diff --git a/packages/voice/tsconfig.json b/packages/voice/tsconfig.json index accd271c10..82d77bf1e5 100644 --- a/packages/voice/tsconfig.json +++ b/packages/voice/tsconfig.json @@ -7,5 +7,5 @@ "noUncheckedIndexedAccess": false, "noUnusedLocals": true }, - "include": ["src"] + "include": ["src", "test"] }