diff --git a/bun.lock b/bun.lock index 6b08db9e5c..1cc1fc02e3 100644 --- a/bun.lock +++ b/bun.lock @@ -986,6 +986,19 @@ "@typescript/native-preview": "catalog:", }, }, + "packages/voice": { + "name": "@opencode-ai/voice", + "version": "0.0.0", + "dependencies": { + "@opencode-ai/client": "workspace:*", + }, + "devDependencies": { + "@tsconfig/bun": "catalog:", + "@types/bun": "catalog:", + "@typescript/native-preview": "catalog:", + "typescript": "catalog:", + }, + }, "packages/web": { "name": "@opencode-ai/web", "version": "1.18.4", @@ -2077,6 +2090,8 @@ "@opencode-ai/util": ["@opencode-ai/util@workspace:packages/util"], + "@opencode-ai/voice": ["@opencode-ai/voice@workspace:packages/voice"], + "@opencode-ai/web": ["@opencode-ai/web@workspace:packages/web"], "@opencode-ai/www": ["@opencode-ai/www@workspace:packages/www"], diff --git a/packages/voice/package.json b/packages/voice/package.json new file mode 100644 index 0000000000..917fa23194 --- /dev/null +++ b/packages/voice/package.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json.schemastore.org/package.json", + "name": "@opencode-ai/voice", + "version": "0.0.0", + "private": true, + "description": "Prototype voice control for OpenCode via the OpenAI Realtime API", + "type": "module", + "scripts": { + "spike": "bun src/spike.ts", + "typecheck": "tsgo --noEmit" + }, + "devDependencies": { + "@tsconfig/bun": "catalog:", + "@types/bun": "catalog:", + "@typescript/native-preview": "catalog:", + "typescript": "catalog:" + }, + "dependencies": { + "@opencode-ai/client": "workspace:*" + } +} diff --git a/packages/voice/src/spike.ts b/packages/voice/src/spike.ts new file mode 100644 index 0000000000..f529ec2f7d --- /dev/null +++ b/packages/voice/src/spike.ts @@ -0,0 +1,373 @@ +#!/usr/bin/env bun +// Voice control spike: bridges the local microphone and speaker to the OpenAI +// Realtime API (gpt-realtime-2.1) and exposes OpenCode session control as +// realtime function tools, so you can drive OpenCode with your voice. +// +// Usage: +// opencode serve (or note the URL of a running server) +// 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- \ +// bun packages/voice/src/spike.ts --server http://localhost:4096 [--directory /path/to/project] +// +// Requires sox (`brew install sox`) for mic capture (`rec`) and playback (`play`). + +import { parseArgs } from "node:util" +import { OpenCode } from "@opencode-ai/client/promise" +import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client/promise" + +const args = parseArgs({ + options: { + server: { type: "string" }, + directory: { type: "string", default: process.cwd() }, + model: { type: "string", default: "gpt-realtime-2.1" }, + voice: { type: "string", default: "marin" }, + // Text mode: send one typed message instead of opening the microphone, + // print the reply, and exit. Useful for smoke-testing the tool loop. + text: { type: "string" }, + }, +}).values + +if (!args.server) { + console.error("Usage: bun src/spike.ts --server http://localhost:PORT [--directory /path]") + console.error("Start a server first with `opencode serve` and pass its URL.") + process.exit(1) +} +const apiKey = process.env["OPENAI_API_KEY"] +if (!apiKey) { + console.error("OPENAI_API_KEY is required. Run via:") + console.error(` 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- bun src/spike.ts ...`) + process.exit(1) +} + +const password = process.env["OPENCODE_SERVER_PASSWORD"] +const client = OpenCode.make({ + baseUrl: args.server, + headers: password ? { Authorization: "Basic " + btoa("opencode:" + password) } : undefined, +}) +const health = await client.health.get() +console.log(`opencode server ${args.server} (version ${health.version})`) +console.log(`project directory: ${args.directory}`) + +// --------------------------------------------------------------------------- +// OpenCode tool surface exposed to the realtime model +// --------------------------------------------------------------------------- + +let activeSessionID: string | undefined +let lastPromptAt = 0 + +function assistantText(message: SessionMessageAssistant) { + return message.content + .filter((part) => part.type === "text") + .map((part) => part.text) + .join("\n") +} + +function latestAssistant(messages: ReadonlyArray) { + return messages.find((message): message is SessionMessageAssistant => message.type === "assistant") +} + +async function requireSession() { + if (activeSessionID) return activeSessionID + const created = await client.session.create({ location: { directory: args.directory! } }) + activeSessionID = created.id + console.log(`\n[voice] created session ${activeSessionID}`) + return created.id +} + +const truncate = (text: string, max = 2000) => (text.length > max ? text.slice(0, max) + "…" : text) + +const toolHandlers: Record) => Promise> = { + list_sessions: async (input) => { + const limit = typeof input["limit"] === "number" ? input["limit"] : 10 + const sessions = await client.session.list({ directory: args.directory, limit, order: "desc" }) + return sessions.data.map((session) => ({ + id: session.id, + title: session.title, + updated: new Date(session.time.updated).toISOString(), + active: session.id === activeSessionID, + })) + }, + create_session: async () => { + activeSessionID = undefined + return { sessionID: await requireSession() } + }, + select_session: async (input) => { + const sessionID = input["session_id"] + if (typeof sessionID !== "string") return { error: "session_id is required" } + activeSessionID = sessionID + return { sessionID, active: true } + }, + prompt_session: async (input) => { + const text = input["text"] + if (typeof text !== "string" || text.length === 0) return { error: "text is required" } + const sessionID = await requireSession() + lastPromptAt = Date.now() + await client.session.prompt({ sessionID, text }) + console.log(`\n[voice] prompted ${sessionID}: ${text}`) + return { sessionID, admitted: true, hint: "Work runs in the background. Use check_session or wait_for_reply." } + }, + check_session: async () => { + if (!activeSessionID) return { error: "no active session" } + const messages = await client.message.list({ sessionID: activeSessionID, order: "desc", limit: 20 }) + const assistant = latestAssistant(messages.data) + if (!assistant) return { status: "no assistant reply yet" } + const tools = assistant.content.filter((part) => part.type === "tool").map((part) => part.name) + return { + status: assistant.time.completed ? "completed" : "working", + finish: assistant.finish, + runningTools: tools, + text: truncate(assistantText(assistant)), + } + }, + wait_for_reply: async (input) => { + if (!activeSessionID) return { error: "no active session" } + const timeout = typeof input["timeout_seconds"] === "number" ? input["timeout_seconds"] : 60 + const deadline = Date.now() + timeout * 1000 + while (Date.now() < deadline) { + const messages = await client.message.list({ sessionID: activeSessionID, order: "desc", limit: 20 }) + const assistant = latestAssistant(messages.data) + const done = + assistant !== undefined && + assistant.time.created >= lastPromptAt && + assistant.time.completed !== undefined && + assistant.finish === "stop" + if (done) return { status: "completed", text: truncate(assistantText(assistant)) } + await Bun.sleep(1000) + } + return { status: "timeout", hint: "Still working. Check again with check_session." } + }, + interrupt_session: async () => { + if (!activeSessionID) return { error: "no active session" } + await client.session.interrupt({ sessionID: activeSessionID }) + return { interrupted: true } + }, +} + +const toolDefinitions = [ + { + type: "function", + name: "list_sessions", + description: "List recent OpenCode sessions in the current project.", + parameters: { + type: "object", + properties: { limit: { type: "number", description: "Max sessions to return (default 10)." } }, + required: [], + }, + }, + { + type: "function", + name: "create_session", + description: "Create a fresh OpenCode session and make it active.", + parameters: { type: "object", properties: {}, required: [] }, + }, + { + type: "function", + name: "select_session", + description: "Make an existing session the active one.", + parameters: { + type: "object", + properties: { session_id: { type: "string", description: "Session ID from list_sessions." } }, + required: ["session_id"], + }, + }, + { + type: "function", + name: "prompt_session", + description: + "Send a task or question to the active OpenCode coding agent. Creates a session if none is active. Returns immediately; the agent works in the background.", + parameters: { + type: "object", + properties: { text: { type: "string", description: "The instruction for the coding agent." } }, + required: ["text"], + }, + }, + { + type: "function", + name: "check_session", + description: "Check what the coding agent is doing right now: status, running tools, and its latest reply text.", + parameters: { type: "object", properties: {}, required: [] }, + }, + { + type: "function", + name: "wait_for_reply", + description: "Block until the coding agent finishes its current turn and return its final reply. Use for quick tasks.", + parameters: { + type: "object", + properties: { timeout_seconds: { type: "number", description: "Max seconds to wait (default 60)." } }, + required: [], + }, + }, + { + type: "function", + name: "interrupt_session", + description: "Stop whatever the coding agent is currently doing in the active session.", + parameters: { type: "object", properties: {}, required: [] }, + }, +] + +const instructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine. +The user talks to you; you control OpenCode with tools. You never write code yourself — the coding agent does. + +Guidelines: +- Keep spoken replies short: one or two sentences. This is a hands-free interface. +- When the user asks for coding work, relay it with prompt_session, phrased clearly for a coding agent. +- For quick questions use wait_for_reply and summarize the agent's answer out loud. +- For longer tasks say the work has started, and use check_session when the user asks for status. +- Summarize agent replies conversationally; never read code, diffs, or file paths aloud verbatim unless asked. +- Confirm before interrupting a session or anything destructive.` + +// --------------------------------------------------------------------------- +// Realtime session over WebSocket +// --------------------------------------------------------------------------- + +type RealtimeItem = { + type?: string + name?: string + call_id?: string + arguments?: string + transcript?: string | null +} +type RealtimeEvent = { + type: string + delta?: string + transcript?: string + item?: RealtimeItem + response?: { output?: ReadonlyArray } + error?: { type?: string; code?: string; message?: string } +} + +const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${args.model}`, { + // Bun extension: custom headers on the WebSocket handshake + headers: { Authorization: `Bearer ${apiKey}` }, +} as unknown as string[]) + +const send = (event: Record) => ws.send(JSON.stringify(event)) + +let recorder: ReturnType | undefined +let player: ReturnType | undefined + +// PCM16 mono 24kHz is the realtime API default; sox handles both directions. +const soxFormat = ["-q", "-t", "raw", "-r", "24000", "-e", "signed-integer", "-b", "16", "-c", "1"] + +async function startMicrophone() { + recorder = Bun.spawn(["rec", ...soxFormat, "-"], { stdout: "pipe", stderr: "ignore" }) + console.log("[voice] microphone live — start talking (ctrl+c to quit)") + for await (const chunk of recorder.stdout as ReadableStream) { + if (ws.readyState !== WebSocket.OPEN) break + send({ type: "input_audio_buffer.append", audio: Buffer.from(chunk).toString("base64") }) + } +} + +function playAudio(base64: string) { + player ??= Bun.spawn(["play", ...soxFormat, "-"], { stdin: "pipe", stderr: "ignore" }) + const stdin = player.stdin as import("bun").FileSink + stdin.write(Buffer.from(base64, "base64")) + stdin.flush() +} + +function flushPlayback() { + player?.kill() + player = undefined +} + +const createResponse = () => + send(args.text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" }) + +let inflightTools = 0 + +async function handleFunctionCall(item: RealtimeItem) { + inflightTools += 1 + const handler = item.name ? toolHandlers[item.name] : undefined + const output = handler + ? await handler(JSON.parse(item.arguments ?? "{}")).catch((error) => ({ error: String(error) })) + : { error: `unknown tool ${item.name}` } + console.log(`\n[tool] ${item.name} -> ${truncate(JSON.stringify(output), 300)}`) + send({ + type: "conversation.item.create", + item: { type: "function_call_output", call_id: item.call_id, output: JSON.stringify(output) }, + }) + createResponse() + inflightTools -= 1 +} + +ws.addEventListener("open", () => { + console.log(`[voice] connected to ${args.model}`) + send({ + type: "session.update", + session: { + type: "realtime", + instructions, + tools: toolDefinitions, + tool_choice: "auto", + audio: { output: { voice: args.voice } }, + }, + }) + // Optional extras sent separately so a shape mismatch can't reject the core session config. + send({ + type: "session.update", + session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } }, + }) +}) + +ws.addEventListener("message", (event) => { + const data = JSON.parse(String(event.data)) as RealtimeEvent + switch (data.type) { + case "session.created": + if (!args.text) { + void startMicrophone() + break + } + console.log(`You (typed): ${args.text}`) + send({ + type: "conversation.item.create", + item: { type: "message", role: "user", content: [{ type: "input_text", text: args.text }] }, + }) + createResponse() + break + case "response.output_text.delta": + process.stdout.write(data.delta ?? "") + break + case "response.done": { + const calledFunction = data.response?.output?.some((item) => item.type === "function_call") ?? false + if (args.text && !calledFunction && inflightTools === 0) { + process.stdout.write("\n") + shutdown() + } + break + } + case "input_audio_buffer.speech_started": + flushPlayback() // barge-in: stop speaking as soon as the user does + break + case "conversation.item.input_audio_transcription.completed": + console.log(`\nYou: ${data.transcript ?? ""}`) + break + case "response.output_audio.delta": + if (data.delta) playAudio(data.delta) + break + case "response.output_audio_transcript.delta": + process.stdout.write(data.delta ?? "") + break + case "response.output_audio_transcript.done": + process.stdout.write("\n") + break + case "response.output_item.done": + if (data.item?.type === "function_call") void handleFunctionCall(data.item) + break + case "error": + console.error(`\n[realtime error] ${data.error?.code}: ${data.error?.message}`) + break + } +}) + +ws.addEventListener("close", (event) => { + console.log(`\n[voice] realtime connection closed (${event.code})`) + shutdown() +}) + +function shutdown() { + recorder?.kill() + player?.kill() + if (ws.readyState === WebSocket.OPEN) ws.close() + process.exit(0) +} + +process.on("SIGINT", shutdown) diff --git a/packages/voice/tsconfig.json b/packages/voice/tsconfig.json new file mode 100644 index 0000000000..0f2fdf0ce8 --- /dev/null +++ b/packages/voice/tsconfig.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "@tsconfig/bun/tsconfig.json", + "compilerOptions": { + "noUncheckedIndexedAccess": false, + "noUnusedLocals": true + }, + "include": ["src"] +}