ceos/opencode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(voice): realtime voice control spike

Browse source
This commit is contained in:
Kit Langton 2026-07-26 11:08:47 -04:00
commit 699477755f
4 changed files with 418 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

15
bun.lock
View file

@ -986,6 +986,19 @@
"@typescript/native-preview": "catalog:",
},
},
"packages/voice": {
"name": "@opencode-ai/voice",
"version": "0.0.0",
"dependencies": {
"@opencode-ai/client": "workspace:*",
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:",
},
},
"packages/web": {
"name": "@opencode-ai/web",
"version": "1.18.4",
@ -2077,6 +2090,8 @@
"@opencode-ai/util": ["@opencode-ai/util@workspace:packages/util"],
"@opencode-ai/voice": ["@opencode-ai/voice@workspace:packages/voice"],
"@opencode-ai/web": ["@opencode-ai/web@workspace:packages/web"],
"@opencode-ai/www": ["@opencode-ai/www@workspace:packages/www"],

21
packages/voice/package.json Normal file
View file

@ -0,0 +1,21 @@
{
"$schema": "https://json.schemastore.org/package.json",
"name": "@opencode-ai/voice",
"version": "0.0.0",
"private": true,
"description": "Prototype voice control for OpenCode via the OpenAI Realtime API",
"type": "module",
"scripts": {
"spike": "bun src/spike.ts",
"typecheck": "tsgo --noEmit"
},
"devDependencies": {
"@tsconfig/bun": "catalog:",
"@types/bun": "catalog:",
"@typescript/native-preview": "catalog:",
"typescript": "catalog:"
},
"dependencies": {
"@opencode-ai/client": "workspace:*"
}
}

373
packages/voice/src/spike.ts Normal file
View file

@ -0,0 +1,373 @@
#!/usr/bin/env bun
// Voice control spike: bridges the local microphone and speaker to the OpenAI
// Realtime API (gpt-realtime-2.1) and exposes OpenCode session control as
// realtime function tools, so you can drive OpenCode with your voice.
//
// Usage:
// opencode serve (or note the URL of a running server)
// 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- \
// bun packages/voice/src/spike.ts --server http://localhost:4096 [--directory /path/to/project]
//
// Requires sox (`brew install sox`) for mic capture (`rec`) and playback (`play`).
import { parseArgs } from "node:util"
import { OpenCode } from "@opencode-ai/client/promise"
import type { SessionMessageAssistant, SessionMessageInfo } from "@opencode-ai/client/promise"
const args = parseArgs({
options: {
server: { type: "string" },
directory: { type: "string", default: process.cwd() },
model: { type: "string", default: "gpt-realtime-2.1" },
voice: { type: "string", default: "marin" },
// Text mode: send one typed message instead of opening the microphone,
// print the reply, and exit. Useful for smoke-testing the tool loop.
text: { type: "string" },
},
}).values
if (!args.server) {
console.error("Usage: bun src/spike.ts --server http://localhost:PORT [--directory /path]")
console.error("Start a server first with `opencode serve` and pass its URL.")
process.exit(1)
}
const apiKey = process.env["OPENAI_API_KEY"]
if (!apiKey) {
console.error("OPENAI_API_KEY is required. Run via:")
console.error(` 2password run --env "OPENAI_API_KEY=op://Personal/OpenAI API Key/credential" -- bun src/spike.ts ...`)
process.exit(1)
}
const password = process.env["OPENCODE_SERVER_PASSWORD"]
const client = OpenCode.make({
baseUrl: args.server,
headers: password ? { Authorization: "Basic " + btoa("opencode:" + password) } : undefined,
})
const health = await client.health.get()
console.log(`opencode server ${args.server} (version ${health.version})`)
console.log(`project directory: ${args.directory}`)
// ---------------------------------------------------------------------------
// OpenCode tool surface exposed to the realtime model
// ---------------------------------------------------------------------------
let activeSessionID: string | undefined
let lastPromptAt = 0
function assistantText(message: SessionMessageAssistant) {
return message.content
.filter((part) => part.type === "text")
.map((part) => part.text)
.join("\n")
}
function latestAssistant(messages: ReadonlyArray<SessionMessageInfo>) {
return messages.find((message): message is SessionMessageAssistant => message.type === "assistant")
}
async function requireSession() {
if (activeSessionID) return activeSessionID
const created = await client.session.create({ location: { directory: args.directory! } })
activeSessionID = created.id
console.log(`\n[voice] created session ${activeSessionID}`)
return created.id
}
const truncate = (text: string, max = 2000) => (text.length > max ? text.slice(0, max) + "…" : text)
const toolHandlers: Record<string, (input: Record<string, unknown>) => Promise<unknown>> = {
list_sessions: async (input) => {
const limit = typeof input["limit"] === "number" ? input["limit"] : 10
const sessions = await client.session.list({ directory: args.directory, limit, order: "desc" })
return sessions.data.map((session) => ({
id: session.id,
title: session.title,
updated: new Date(session.time.updated).toISOString(),
active: session.id === activeSessionID,
}))
},
create_session: async () => {
activeSessionID = undefined
return { sessionID: await requireSession() }
},
select_session: async (input) => {
const sessionID = input["session_id"]
if (typeof sessionID !== "string") return { error: "session_id is required" }
activeSessionID = sessionID
return { sessionID, active: true }
},
prompt_session: async (input) => {
const text = input["text"]
if (typeof text !== "string" || text.length === 0) return { error: "text is required" }
const sessionID = await requireSession()
lastPromptAt = Date.now()
await client.session.prompt({ sessionID, text })
console.log(`\n[voice] prompted ${sessionID}: ${text}`)
return { sessionID, admitted: true, hint: "Work runs in the background. Use check_session or wait_for_reply." }
},
check_session: async () => {
if (!activeSessionID) return { error: "no active session" }
const messages = await client.message.list({ sessionID: activeSessionID, order: "desc", limit: 20 })
const assistant = latestAssistant(messages.data)
if (!assistant) return { status: "no assistant reply yet" }
const tools = assistant.content.filter((part) => part.type === "tool").map((part) => part.name)
return {
status: assistant.time.completed ? "completed" : "working",
finish: assistant.finish,
runningTools: tools,
text: truncate(assistantText(assistant)),
}
},
wait_for_reply: async (input) => {
if (!activeSessionID) return { error: "no active session" }
const timeout = typeof input["timeout_seconds"] === "number" ? input["timeout_seconds"] : 60
const deadline = Date.now() + timeout * 1000
while (Date.now() < deadline) {
const messages = await client.message.list({ sessionID: activeSessionID, order: "desc", limit: 20 })
const assistant = latestAssistant(messages.data)
const done =
assistant !== undefined &&
assistant.time.created >= lastPromptAt &&
assistant.time.completed !== undefined &&
assistant.finish === "stop"
if (done) return { status: "completed", text: truncate(assistantText(assistant)) }
await Bun.sleep(1000)
}
return { status: "timeout", hint: "Still working. Check again with check_session." }
},
interrupt_session: async () => {
if (!activeSessionID) return { error: "no active session" }
await client.session.interrupt({ sessionID: activeSessionID })
return { interrupted: true }
},
}
const toolDefinitions = [
{
type: "function",
name: "list_sessions",
description: "List recent OpenCode sessions in the current project.",
parameters: {
type: "object",
properties: { limit: { type: "number", description: "Max sessions to return (default 10)." } },
required: [],
},
},
{
type: "function",
name: "create_session",
description: "Create a fresh OpenCode session and make it active.",
parameters: { type: "object", properties: {}, required: [] },
},
{
type: "function",
name: "select_session",
description: "Make an existing session the active one.",
parameters: {
type: "object",
properties: { session_id: { type: "string", description: "Session ID from list_sessions." } },
required: ["session_id"],
},
},
{
type: "function",
name: "prompt_session",
description:
"Send a task or question to the active OpenCode coding agent. Creates a session if none is active. Returns immediately; the agent works in the background.",
parameters: {
type: "object",
properties: { text: { type: "string", description: "The instruction for the coding agent." } },
required: ["text"],
},
},
{
type: "function",
name: "check_session",
description: "Check what the coding agent is doing right now: status, running tools, and its latest reply text.",
parameters: { type: "object", properties: {}, required: [] },
},
{
type: "function",
name: "wait_for_reply",
description: "Block until the coding agent finishes its current turn and return its final reply. Use for quick tasks.",
parameters: {
type: "object",
properties: { timeout_seconds: { type: "number", description: "Max seconds to wait (default 60)." } },
required: [],
},
},
{
type: "function",
name: "interrupt_session",
description: "Stop whatever the coding agent is currently doing in the active session.",
parameters: { type: "object", properties: {}, required: [] },
},
]
const instructions = `You are the voice interface to OpenCode, a coding agent running on the user's machine.
The user talks to you; you control OpenCode with tools. You never write code yourself — the coding agent does.
Guidelines:
- Keep spoken replies short: one or two sentences. This is a hands-free interface.
- When the user asks for coding work, relay it with prompt_session, phrased clearly for a coding agent.
- For quick questions use wait_for_reply and summarize the agent's answer out loud.
- For longer tasks say the work has started, and use check_session when the user asks for status.
- Summarize agent replies conversationally; never read code, diffs, or file paths aloud verbatim unless asked.
- Confirm before interrupting a session or anything destructive.`
// ---------------------------------------------------------------------------
// Realtime session over WebSocket
// ---------------------------------------------------------------------------
type RealtimeItem = {
type?: string
name?: string
call_id?: string
arguments?: string
transcript?: string | null
}
type RealtimeEvent = {
type: string
delta?: string
transcript?: string
item?: RealtimeItem
response?: { output?: ReadonlyArray<RealtimeItem> }
error?: { type?: string; code?: string; message?: string }
}
const ws = new WebSocket(`wss://api.openai.com/v1/realtime?model=${args.model}`, {
// Bun extension: custom headers on the WebSocket handshake
headers: { Authorization: `Bearer ${apiKey}` },
} as unknown as string[])
const send = (event: Record<string, unknown>) => ws.send(JSON.stringify(event))
let recorder: ReturnType<typeof Bun.spawn> | undefined
let player: ReturnType<typeof Bun.spawn> | undefined
// PCM16 mono 24kHz is the realtime API default; sox handles both directions.
const soxFormat = ["-q", "-t", "raw", "-r", "24000", "-e", "signed-integer", "-b", "16", "-c", "1"]
async function startMicrophone() {
recorder = Bun.spawn(["rec", ...soxFormat, "-"], { stdout: "pipe", stderr: "ignore" })
console.log("[voice] microphone live — start talking (ctrl+c to quit)")
for await (const chunk of recorder.stdout as ReadableStream<Uint8Array>) {
if (ws.readyState !== WebSocket.OPEN) break
send({ type: "input_audio_buffer.append", audio: Buffer.from(chunk).toString("base64") })
}
}
function playAudio(base64: string) {
player ??= Bun.spawn(["play", ...soxFormat, "-"], { stdin: "pipe", stderr: "ignore" })
const stdin = player.stdin as import("bun").FileSink
stdin.write(Buffer.from(base64, "base64"))
stdin.flush()
}
function flushPlayback() {
player?.kill()
player = undefined
}
const createResponse = () =>
send(args.text ? { type: "response.create", response: { output_modalities: ["text"] } } : { type: "response.create" })
let inflightTools = 0
async function handleFunctionCall(item: RealtimeItem) {
inflightTools += 1
const handler = item.name ? toolHandlers[item.name] : undefined
const output = handler
? await handler(JSON.parse(item.arguments ?? "{}")).catch((error) => ({ error: String(error) }))
: { error: `unknown tool ${item.name}` }
console.log(`\n[tool] ${item.name} -> ${truncate(JSON.stringify(output), 300)}`)
send({
type: "conversation.item.create",
item: { type: "function_call_output", call_id: item.call_id, output: JSON.stringify(output) },
})
createResponse()
inflightTools -= 1
}
ws.addEventListener("open", () => {
console.log(`[voice] connected to ${args.model}`)
send({
type: "session.update",
session: {
type: "realtime",
instructions,
tools: toolDefinitions,
tool_choice: "auto",
audio: { output: { voice: args.voice } },
},
})
// Optional extras sent separately so a shape mismatch can't reject the core session config.
send({
type: "session.update",
session: { type: "realtime", audio: { input: { transcription: { model: "whisper-1" } } } },
})
})
ws.addEventListener("message", (event) => {
const data = JSON.parse(String(event.data)) as RealtimeEvent
switch (data.type) {
case "session.created":
if (!args.text) {
void startMicrophone()
break
}
console.log(`You (typed): ${args.text}`)
send({
type: "conversation.item.create",
item: { type: "message", role: "user", content: [{ type: "input_text", text: args.text }] },
})
createResponse()
break
case "response.output_text.delta":
process.stdout.write(data.delta ?? "")
break
case "response.done": {
const calledFunction = data.response?.output?.some((item) => item.type === "function_call") ?? false
if (args.text && !calledFunction && inflightTools === 0) {
process.stdout.write("\n")
shutdown()
}
break
}
case "input_audio_buffer.speech_started":
flushPlayback() // barge-in: stop speaking as soon as the user does
break
case "conversation.item.input_audio_transcription.completed":
console.log(`\nYou: ${data.transcript ?? ""}`)
break
case "response.output_audio.delta":
if (data.delta) playAudio(data.delta)
break
case "response.output_audio_transcript.delta":
process.stdout.write(data.delta ?? "")
break
case "response.output_audio_transcript.done":
process.stdout.write("\n")
break
case "response.output_item.done":
if (data.item?.type === "function_call") void handleFunctionCall(data.item)
break
case "error":
console.error(`\n[realtime error] ${data.error?.code}: ${data.error?.message}`)
break
}
})
ws.addEventListener("close", (event) => {
console.log(`\n[voice] realtime connection closed (${event.code})`)
shutdown()
})
function shutdown() {
recorder?.kill()
player?.kill()
if (ws.readyState === WebSocket.OPEN) ws.close()
process.exit(0)
}
process.on("SIGINT", shutdown)

9
packages/voice/tsconfig.json Normal file
View file

@ -0,0 +1,9 @@
{
"$schema": "https://json.schemastore.org/tsconfig",
"extends": "@tsconfig/bun/tsconfig.json",
"compilerOptions": {
"noUncheckedIndexedAccess": false,
"noUnusedLocals": true
},
"include": ["src"]
}