diff --git a/packages/voice/src/duplex-audio.swift b/packages/voice/src/duplex-audio.swift index eb689645cd..ecee6b935e 100644 --- a/packages/voice/src/duplex-audio.swift +++ b/packages/voice/src/duplex-audio.swift @@ -1,14 +1,24 @@ // Full-duplex terminal audio bridge with Apple voice processing (AEC). // // stdin <- raw PCM16 mono 24kHz to play through the speakers -// stdout -> raw PCM16 mono 24kHz captured from the microphone, -// echo-cancelled against the audio this process plays +// stdout -> raw PCM16 mono 24kHz captured from the microphone // SIGUSR1: drop any queued speaker audio (barge-in flush) // +// Two independent engines: attaching a speaker source to a voice-processed +// engine silently kills its input tap, so playback runs on its own plain +// engine. Voice processing (echo cancellation) is attempted on the input +// engine and abandoned if the mic delivers nothing: on some devices +// (Bluetooth headsets mid-negotiation) the VP tap never fires. Without VP +// there is no echo cancellation, which is acceptable exactly when it happens: +// headphones have no echo path. +// // Compiled on demand by spike.ts: swiftc -O duplex-audio.swift -o duplex-audio import AVFoundation +let stderr = FileHandle.standardError +func log(_ message: String) { stderr.write(Data("[audio] \(message)\n".utf8)) } + final class SpeakerQueue { private var data = Data() private let lock = NSLock() @@ -37,60 +47,106 @@ final class SpeakerQueue { } let queue = SpeakerQueue() -let engine = AVAudioEngine() +let playFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 24000, channels: 1, interleaved: false)! +let captureFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: 24000, channels: 1, interleaved: true)! -do { - // Enables Apple's voice-processed IO unit (acoustic echo cancellation) on - // both sides of this engine. Conveniently it runs at 24kHz natively. - try engine.inputNode.setVoiceProcessingEnabled(true) -} catch { - FileHandle.standardError.write(Data("voice processing unavailable: \(error)\n".utf8)) - exit(2) -} -if #available(macOS 14.0, *) { - // Don't duck other system audio while the mic is hot. - engine.inputNode.voiceProcessingOtherAudioDuckingConfiguration = .init( - enableAdvancedDucking: false, - duckingLevel: .min - ) -} - -let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 24000, channels: 1, interleaved: false)! - -// Speaker: pull PCM16 from the stdin-fed queue, emit silence when empty. -let source = AVAudioSourceNode(format: format) { _, _, frameCount, audioBufferList -> OSStatus in +// Speaker: its own engine, pulling PCM16 from the stdin-fed queue. +let outputEngine = AVAudioEngine() +let source = AVAudioSourceNode(format: playFormat) { _, _, frameCount, audioBufferList -> OSStatus in let samples = queue.pop(frames: Int(frameCount)) let out = UnsafeMutableAudioBufferListPointer(audioBufferList)[0].mData!.assumingMemoryBound(to: Float.self) for i in 0.. 0 else { return } + guard let mono = AVAudioPCMBuffer(pcmFormat: monoFormat, frameCapacity: buffer.frameLength) else { return } + memcpy(mono.floatChannelData![0], channel, Int(buffer.frameLength) * 4) + mono.frameLength = buffer.frameLength + + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * 24000.0 / micFormat.sampleRate) + 32 + guard let converted = AVAudioPCMBuffer(pcmFormat: captureFormat, frameCapacity: capacity) else { return } + var consumed = false + converter.convert(to: converted, error: nil) { _, status in + if consumed { + status.pointee = .noDataNow + return nil + } + consumed = true + status.pointee = .haveData + return mono + } + guard converted.frameLength > 0, let out = converted.int16ChannelData?[0] else { return } + FileHandle.standardOutput.write(Data(bytes: out, count: Int(converted.frameLength) * 2)) + } + + do { + try engine.start() + } catch { + if voiceProcessing { + log("input engine failed with voice processing (\(error)) — retrying without") + return startInput(voiceProcessing: false) + } + log("input engine failed: \(error)") + exit(2) + } + + // Watchdog: on some devices the voice-processed tap simply never fires. + // Fall back to a plain tap; without VP the mic reliably delivers. + if voiceProcessing { + DispatchQueue.main.asyncAfter(deadline: .now() + 2.5) { + if tapCount > 0 { return } + log("voice-processed mic delivered nothing — restarting without echo cancellation") + startInput(voiceProcessing: false) } - consumed = true - status.pointee = .haveData - return buffer } - guard converted.frameLength > 0, let channel = converted.int16ChannelData?[0] else { return } - FileHandle.standardOutput.write(Data(bytes: channel, count: Int(converted.frameLength) * 2)) } +// Voice processing is opt-in (--aec): it is only needed on speakers, and on +// some machines (observed with Bluetooth headsets active) the VP engine binds +// to the wrong capture device entirely, delivering noise instead of the mic. +startInput(voiceProcessing: CommandLine.arguments.contains("--aec")) + signal(SIGUSR1, SIG_IGN) let flushSignal = DispatchSource.makeSignalSource(signal: SIGUSR1, queue: .main) flushSignal.setEventHandler { queue.flush() } @@ -104,5 +160,4 @@ DispatchQueue.global().async { } } -try engine.start() RunLoop.main.run() diff --git a/packages/voice/src/spike.ts b/packages/voice/src/spike.ts index 00dd23a249..6fc3954bde 100644 --- a/packages/voice/src/spike.ts +++ b/packages/voice/src/spike.ts @@ -25,6 +25,10 @@ const args = parseArgs({ // usable with headphones: on speakers the mic hears the assistant and // interrupts it with its own echo. Default is half-duplex gating. duplex: { type: "boolean", default: false }, + // Enable Apple voice processing (echo cancellation) in the audio helper. + // Needed for full duplex on speakers; harmful with Bluetooth headsets, + // where it can bind the wrong capture device. + speakers: { type: "boolean", default: false }, // Start attached to an existing session instead of creating one lazily. session: { type: "string" }, // Text mode: send one typed message instead of opening the microphone, @@ -307,7 +311,11 @@ const assistantSpeaking = () => Date.now() < playbackEndsAt async function startMicrophone() { if (aecBinary) { - audio = Bun.spawn([aecBinary], { stdin: "pipe", stdout: "pipe", stderr: "ignore" }) + audio = Bun.spawn([aecBinary, ...(args.speakers ? ["--aec"] : [])], { + stdin: "pipe", + stdout: "pipe", + stderr: args.debug ? "inherit" : "ignore", + }) console.log("[voice] echo-cancelled duplex audio live — talk any time, even over the assistant (ctrl+c to quit)") for await (const chunk of audio.stdout as ReadableStream) { if (ws.readyState !== WebSocket.OPEN) break