feat(simulation): stream UI recordings (#35909)

packages/simulation/src/frontend/actions.ts

@@ -1,7 +1,7 @@
-import { mkdtemp } from "node:fs/promises"
+import { mkdir, mkdtemp } from "node:fs/promises"
 import { tmpdir } from "node:os"
-import { join } from "node:path"
-import type { CapturedFrame, CliRenderer, Renderable } from "@opentui/core"
+import { dirname, join } from "node:path"
+import type { CliRenderer, Renderable } from "@opentui/core"
 import { createMockKeys, createMockMouse, type MockInput, type MockMouse } from "@opentui/core/testing"
 import type { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
@@ -104,67 +104,15 @@ export function state(harness: Harness) {
   }
 }
 
-export async function screenshot(harness: Harness) {
+export async function screenshot(harness: Harness, output?: string) {
   await harness.renderOnce()
   const image = SimulationPng.screenshot(harness.renderer)
-  const path = join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png")
+  const path = output ?? join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png")
+  if (output) await mkdir(dirname(output), { recursive: true })
   await Bun.write(path, image.data)
   return path
 }
 
-export function frame(harness: Harness): CapturedFrame {
-  const buffer = harness.renderer.currentRenderBuffer
-  return {
-    cols: buffer.width,
-    rows: buffer.height,
-    cursor: [0, 0],
-    lines: buffer.getSpanLines().map((line) => ({
-      spans: line.spans.map((span) => ({
-        text: span.text,
-        fg: span.fg,
-        bg: span.bg,
-        attributes: span.attributes,
-        width: span.width,
-      })),
-    })),
-  }
-}
-
-export async function video(frames: CapturedFrame[]) {
-  const directory = await mkdtemp(join(tmpdir(), "opencode-drive-recording-"))
-  await Promise.all(
-    frames.map((frame, index) =>
-      Bun.write(
-        join(directory, `frame-${index.toString().padStart(6, "0")}.png`),
-        SimulationPng.screenshotFrame(frame).data,
-      ),
-    ),
-  )
-  const path = join(directory, "recording.mp4")
-  const process = Bun.spawn(
-    [
-      "ffmpeg",
-      "-loglevel",
-      "error",
-      "-framerate",
-      "10",
-      "-i",
-      join(directory, "frame-%06d.png"),
-      "-c:v",
-      "libx264",
-      "-pix_fmt",
-      "yuv420p",
-      "-movflags",
-      "+faststart",
-      "-y",
-      path,
-    ],
-    { stderr: "pipe" },
-  )
-  if ((await process.exited) !== 0) throw new Error(`ffmpeg failed: ${await new Response(process.stderr).text()}`)
-  return path
-}
-
 export async function execute(harness: Harness, action: Action) {
   switch (action.type) {
     case "ui.type":
@@ -180,7 +128,9 @@ export async function execute(harness: Harness, action: Action) {
       harness.mockInput.pressArrow(action.direction)
       break
     case "ui.focus":
-      all(harness.renderer.root).find((item) => item.num === action.target)?.focus()
+      all(harness.renderer.root)
+        .find((item) => item.num === action.target)
+        ?.focus()
       break
     case "ui.click":
       await harness.mockMouse.click(action.x, action.y)

packages/simulation/src/frontend/renderer.ts

@@ -1,21 +1,43 @@
 import type { CliRenderer, CliRendererConfig } from "@opentui/core"
 import { createTestRenderer, type TestRendererSetup } from "@opentui/core/testing"
+import { Timeline } from "../recording"
 
 const setups = new WeakMap<CliRenderer, TestRendererSetup>()
+const recordings = new WeakMap<CliRenderer, Timeline>()
 
 /**
- * Creates the headless simulation renderer: a real CliRenderer backed by an
- * in-memory screen buffer instead of a terminal. The TestRendererSetup is
- * kept module-side (keyed by renderer) so the harness can use the supported
- * testing APIs without app code carrying it around.
+ * Creates a headless renderer with optional recording: a real CliRenderer
+ * backed by an in-memory screen buffer. The TestRendererSetup is kept
+ * module-side so the harness can use supported testing APIs without app
+ * code carrying it around.
  */
-export async function create(options: CliRendererConfig): Promise<CliRenderer> {
+export async function create(options: CliRendererConfig, path?: string): Promise<CliRenderer> {
+  if (!path) {
+    const setup = await createTestRenderer({
+      ...options,
+      width: 100,
+      height: 40,
+    })
+    setups.set(setup.renderer, setup)
+    return setup.renderer
+  }
+  const recording = await Timeline.create(path, 100, 40)
   const setup = await createTestRenderer({
     ...options,
     width: 100,
     height: 40,
+    stdout: recording as unknown as NodeJS.WriteStream,
+    bufferedOutput: "stdout",
+    onDestroy: () => {
+      void recording.finish().catch((error) => process.stderr.write(`Failed to finish UI recording: ${error}\n`))
+      options.onDestroy?.()
+    },
+  }).catch(async (error) => {
+    await recording.finish().catch(() => undefined)
+    throw error
   })
   setups.set(setup.renderer, setup)
+  recordings.set(setup.renderer, recording)
   return setup.renderer
 }
 
@@ -23,4 +45,10 @@ export function setupFor(renderer: CliRenderer): TestRendererSetup | undefined {
   return setups.get(renderer)
 }
 
+export function finish(renderer: CliRenderer) {
+  const recording = recordings.get(renderer)
+  if (!recording) throw new Error("UI recording is not available")
+  return recording.finish()
+}
+
 export * as SimulationRenderer from "./renderer"

packages/simulation/src/frontend/server.ts

@@ -1,4 +1,3 @@
-import type { CapturedFrame } from "@opentui/core"
 import { SimulationProtocol } from "../protocol"
 import { SimulationActions, type Harness } from "./actions"
 
@@ -11,50 +10,20 @@ function parseRequest(input: string | Buffer) {
   return SimulationProtocol.Frontend.decodeRequest(JSON.parse(typeof input === "string" ? input : input.toString()))
 }
 
-interface Recording {
-  readonly frames: CapturedFrame[]
-  readonly timer: ReturnType<typeof setInterval>
-  pending: Promise<void>
-}
-
 async function handle(
   harness: Harness,
   request: SimulationProtocol.Frontend.Request,
-  recording: { current?: Recording },
-  headless: boolean,
+  finishRecording?: () => Promise<string>,
 ) {
   switch (request.method) {
     case "ui.screenshot":
-      return SimulationActions.screenshot(harness)
+      return SimulationActions.screenshot(harness, request.params?.path)
     case "ui.state": {
       return SimulationActions.state(harness)
     }
-    case "ui.start-record": {
-      if (recording.current) throw new Error("UI recording is already active")
-      const frames = [SimulationActions.frame(harness)]
-      const current: Recording = {
-        frames,
-        timer: setInterval(() => {
-          current.pending = current.pending.then(async () => {
-            if (headless) await harness.renderOnce()
-            frames.push(SimulationActions.frame(harness))
-          })
-        }, 100),
-        pending: Promise.resolve(),
-      }
-      recording.current = current
-      return { recording: true }
-    }
-    case "ui.end-record": {
-      if (!recording.current) throw new Error("UI recording is not active")
-      const current = recording.current
-      clearInterval(current.timer)
-      await current.pending
-      if (headless) await harness.renderOnce()
-      current.frames.push(SimulationActions.frame(harness))
-      recording.current = undefined
-      return SimulationActions.video(current.frames)
-    }
+    case "ui.recording.finish":
+      if (!finishRecording) throw new Error("UI recording is not available")
+      return finishRecording()
     case "ui.type":
       return SimulationActions.execute(harness, { type: "ui.type", text: request.params.text })
     case "ui.enter":
@@ -79,14 +48,13 @@ async function handle(
   }
 }
 
-export function start(harness: Harness, endpoint: string, headless: boolean): Server {
+export function start(harness: Harness, endpoint: string, finishRecording?: () => Promise<string>): Server {
   const url = new URL(endpoint)
-  const recording: { current?: Recording } = {}
-  const server = Bun.serve<{ readonly drive: true; readonly headless: boolean }>({
+  const server = Bun.serve<{ readonly drive: true }>({
     hostname: url.hostname,
     port: Number(url.port),
     fetch(request, server) {
-      if (server.upgrade(request, { data: { drive: true, headless } })) return undefined
+      if (server.upgrade(request, { data: { drive: true } })) return undefined
       return new Response("opencode drive ui websocket", { status: 426 })
     },
     websocket: {
@@ -94,7 +62,7 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se
         let request: SimulationProtocol.Frontend.Request | undefined
         try {
           request = parseRequest(message)
-          const result = await handle(harness, request, recording, headless)
+          const result = await handle(harness, request, finishRecording)
           const next = SimulationProtocol.JsonRpc.success(request.id, result)
           if (next) socket.send(JSON.stringify(next))
         } catch (error) {
@@ -106,7 +74,6 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se
   return {
     url: endpoint,
     stop: () => {
-      if (recording.current) clearInterval(recording.current.timer)
       server.stop(true)
     },
   }

packages/simulation/src/frontend/simulation.ts

@@ -14,11 +14,14 @@ import { SimulationServer } from "./server"
  */
 export async function create(options: CliRendererConfig): Promise<CliRenderer> {
   const headless = process.env.OPENCODE_DRIVE_RENDERER === "headless"
-  const renderer = headless ? await SimulationRenderer.create(options) : await createCliRenderer(options)
+  const manifest = DriveManifest.resolve()
+  const renderer = headless
+    ? await SimulationRenderer.create(options, manifest.recording?.timeline)
+    : await createCliRenderer(options)
   const server = SimulationServer.start(
     SimulationActions.createHarness(renderer),
-    DriveManifest.resolve().endpoints.ui,
-    headless,
+    manifest.endpoints.ui,
+    headless && manifest.recording ? () => SimulationRenderer.finish(renderer) : undefined,
   )
   process.stderr.write(`opencode drive ui websocket: ${server.url}\n`)
   renderer.once("destroy", () => server.stop())