feat(simulation): stream UI recordings (#35909)

packages/simulation/package.json

@@ -10,7 +10,8 @@
     "./backend/*": "./src/backend/*.ts",
     "./frontend": "./src/frontend/simulation.ts",
     "./frontend/*": "./src/frontend/*.ts",
-    "./protocol": "./src/protocol/index.ts"
+    "./protocol": "./src/protocol/index.ts",
+    "./recording": "./src/recording.ts"
   },
   "scripts": {
     "typecheck": "tsgo --noEmit"

packages/simulation/src/frontend/actions.ts

@@ -1,7 +1,7 @@
-import { mkdtemp } from "node:fs/promises"
+import { mkdir, mkdtemp } from "node:fs/promises"
 import { tmpdir } from "node:os"
-import { join } from "node:path"
-import type { CapturedFrame, CliRenderer, Renderable } from "@opentui/core"
+import { dirname, join } from "node:path"
+import type { CliRenderer, Renderable } from "@opentui/core"
 import { createMockKeys, createMockMouse, type MockInput, type MockMouse } from "@opentui/core/testing"
 import type { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
@@ -104,67 +104,15 @@ export function state(harness: Harness) {
   }
 }
 
-export async function screenshot(harness: Harness) {
+export async function screenshot(harness: Harness, output?: string) {
   await harness.renderOnce()
   const image = SimulationPng.screenshot(harness.renderer)
-  const path = join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png")
+  const path = output ?? join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png")
+  if (output) await mkdir(dirname(output), { recursive: true })
   await Bun.write(path, image.data)
   return path
 }
 
-export function frame(harness: Harness): CapturedFrame {
-  const buffer = harness.renderer.currentRenderBuffer
-  return {
-    cols: buffer.width,
-    rows: buffer.height,
-    cursor: [0, 0],
-    lines: buffer.getSpanLines().map((line) => ({
-      spans: line.spans.map((span) => ({
-        text: span.text,
-        fg: span.fg,
-        bg: span.bg,
-        attributes: span.attributes,
-        width: span.width,
-      })),
-    })),
-  }
-}
-
-export async function video(frames: CapturedFrame[]) {
-  const directory = await mkdtemp(join(tmpdir(), "opencode-drive-recording-"))
-  await Promise.all(
-    frames.map((frame, index) =>
-      Bun.write(
-        join(directory, `frame-${index.toString().padStart(6, "0")}.png`),
-        SimulationPng.screenshotFrame(frame).data,
-      ),
-    ),
-  )
-  const path = join(directory, "recording.mp4")
-  const process = Bun.spawn(
-    [
-      "ffmpeg",
-      "-loglevel",
-      "error",
-      "-framerate",
-      "10",
-      "-i",
-      join(directory, "frame-%06d.png"),
-      "-c:v",
-      "libx264",
-      "-pix_fmt",
-      "yuv420p",
-      "-movflags",
-      "+faststart",
-      "-y",
-      path,
-    ],
-    { stderr: "pipe" },
-  )
-  if ((await process.exited) !== 0) throw new Error(`ffmpeg failed: ${await new Response(process.stderr).text()}`)
-  return path
-}
-
 export async function execute(harness: Harness, action: Action) {
   switch (action.type) {
     case "ui.type":
@@ -180,7 +128,9 @@ export async function execute(harness: Harness, action: Action) {
       harness.mockInput.pressArrow(action.direction)
       break
     case "ui.focus":
-      all(harness.renderer.root).find((item) => item.num === action.target)?.focus()
+      all(harness.renderer.root)
+        .find((item) => item.num === action.target)
+        ?.focus()
       break
     case "ui.click":
       await harness.mockMouse.click(action.x, action.y)

packages/simulation/src/frontend/renderer.ts

@@ -1,21 +1,43 @@
 import type { CliRenderer, CliRendererConfig } from "@opentui/core"
 import { createTestRenderer, type TestRendererSetup } from "@opentui/core/testing"
+import { Timeline } from "../recording"
 
 const setups = new WeakMap<CliRenderer, TestRendererSetup>()
+const recordings = new WeakMap<CliRenderer, Timeline>()
 
 /**
- * Creates the headless simulation renderer: a real CliRenderer backed by an
- * in-memory screen buffer instead of a terminal. The TestRendererSetup is
- * kept module-side (keyed by renderer) so the harness can use the supported
- * testing APIs without app code carrying it around.
+ * Creates a headless renderer with optional recording: a real CliRenderer
+ * backed by an in-memory screen buffer. The TestRendererSetup is kept
+ * module-side so the harness can use supported testing APIs without app
+ * code carrying it around.
  */
-export async function create(options: CliRendererConfig): Promise<CliRenderer> {
+export async function create(options: CliRendererConfig, path?: string): Promise<CliRenderer> {
+  if (!path) {
+    const setup = await createTestRenderer({
+      ...options,
+      width: 100,
+      height: 40,
+    })
+    setups.set(setup.renderer, setup)
+    return setup.renderer
+  }
+  const recording = await Timeline.create(path, 100, 40)
   const setup = await createTestRenderer({
     ...options,
     width: 100,
     height: 40,
+    stdout: recording as unknown as NodeJS.WriteStream,
+    bufferedOutput: "stdout",
+    onDestroy: () => {
+      void recording.finish().catch((error) => process.stderr.write(`Failed to finish UI recording: ${error}\n`))
+      options.onDestroy?.()
+    },
+  }).catch(async (error) => {
+    await recording.finish().catch(() => undefined)
+    throw error
   })
   setups.set(setup.renderer, setup)
+  recordings.set(setup.renderer, recording)
   return setup.renderer
 }
 
@@ -23,4 +45,10 @@ export function setupFor(renderer: CliRenderer): TestRendererSetup | undefined {
   return setups.get(renderer)
 }
 
+export function finish(renderer: CliRenderer) {
+  const recording = recordings.get(renderer)
+  if (!recording) throw new Error("UI recording is not available")
+  return recording.finish()
+}
+
 export * as SimulationRenderer from "./renderer"

packages/simulation/src/frontend/server.ts

@@ -1,4 +1,3 @@
-import type { CapturedFrame } from "@opentui/core"
 import { SimulationProtocol } from "../protocol"
 import { SimulationActions, type Harness } from "./actions"
 
@@ -11,50 +10,20 @@ function parseRequest(input: string | Buffer) {
   return SimulationProtocol.Frontend.decodeRequest(JSON.parse(typeof input === "string" ? input : input.toString()))
 }
 
-interface Recording {
-  readonly frames: CapturedFrame[]
-  readonly timer: ReturnType<typeof setInterval>
-  pending: Promise<void>
-}
-
 async function handle(
   harness: Harness,
   request: SimulationProtocol.Frontend.Request,
-  recording: { current?: Recording },
-  headless: boolean,
+  finishRecording?: () => Promise<string>,
 ) {
   switch (request.method) {
     case "ui.screenshot":
-      return SimulationActions.screenshot(harness)
+      return SimulationActions.screenshot(harness, request.params?.path)
     case "ui.state": {
       return SimulationActions.state(harness)
     }
-    case "ui.start-record": {
-      if (recording.current) throw new Error("UI recording is already active")
-      const frames = [SimulationActions.frame(harness)]
-      const current: Recording = {
-        frames,
-        timer: setInterval(() => {
-          current.pending = current.pending.then(async () => {
-            if (headless) await harness.renderOnce()
-            frames.push(SimulationActions.frame(harness))
-          })
-        }, 100),
-        pending: Promise.resolve(),
-      }
-      recording.current = current
-      return { recording: true }
-    }
-    case "ui.end-record": {
-      if (!recording.current) throw new Error("UI recording is not active")
-      const current = recording.current
-      clearInterval(current.timer)
-      await current.pending
-      if (headless) await harness.renderOnce()
-      current.frames.push(SimulationActions.frame(harness))
-      recording.current = undefined
-      return SimulationActions.video(current.frames)
-    }
+    case "ui.recording.finish":
+      if (!finishRecording) throw new Error("UI recording is not available")
+      return finishRecording()
     case "ui.type":
       return SimulationActions.execute(harness, { type: "ui.type", text: request.params.text })
     case "ui.enter":
@@ -79,14 +48,13 @@ async function handle(
   }
 }
 
-export function start(harness: Harness, endpoint: string, headless: boolean): Server {
+export function start(harness: Harness, endpoint: string, finishRecording?: () => Promise<string>): Server {
   const url = new URL(endpoint)
-  const recording: { current?: Recording } = {}
-  const server = Bun.serve<{ readonly drive: true; readonly headless: boolean }>({
+  const server = Bun.serve<{ readonly drive: true }>({
     hostname: url.hostname,
     port: Number(url.port),
     fetch(request, server) {
-      if (server.upgrade(request, { data: { drive: true, headless } })) return undefined
+      if (server.upgrade(request, { data: { drive: true } })) return undefined
       return new Response("opencode drive ui websocket", { status: 426 })
     },
     websocket: {
@@ -94,7 +62,7 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se
         let request: SimulationProtocol.Frontend.Request | undefined
         try {
           request = parseRequest(message)
-          const result = await handle(harness, request, recording, headless)
+          const result = await handle(harness, request, finishRecording)
           const next = SimulationProtocol.JsonRpc.success(request.id, result)
           if (next) socket.send(JSON.stringify(next))
         } catch (error) {
@@ -106,7 +74,6 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se
   return {
     url: endpoint,
     stop: () => {
-      if (recording.current) clearInterval(recording.current.timer)
       server.stop(true)
     },
   }

packages/simulation/src/frontend/simulation.ts

@@ -14,11 +14,14 @@ import { SimulationServer } from "./server"
  */
 export async function create(options: CliRendererConfig): Promise<CliRenderer> {
   const headless = process.env.OPENCODE_DRIVE_RENDERER === "headless"
-  const renderer = headless ? await SimulationRenderer.create(options) : await createCliRenderer(options)
+  const manifest = DriveManifest.resolve()
+  const renderer = headless
+    ? await SimulationRenderer.create(options, manifest.recording?.timeline)
+    : await createCliRenderer(options)
   const server = SimulationServer.start(
     SimulationActions.createHarness(renderer),
-    DriveManifest.resolve().endpoints.ui,
-    headless,
+    manifest.endpoints.ui,
+    headless && manifest.recording ? () => SimulationRenderer.finish(renderer) : undefined,
   )
   process.stderr.write(`opencode drive ui websocket: ${server.url}\n`)
   renderer.once("destroy", () => server.stop())

packages/simulation/src/manifest.ts

@@ -1,12 +1,15 @@
 import { existsSync, readFileSync } from "node:fs"
 import { homedir } from "node:os"
-import { join } from "node:path"
+import { isAbsolute, join } from "node:path"
 
 export interface Manifest {
   readonly endpoints: {
     readonly ui: string
     readonly backend: string
   }
+  readonly recording?: {
+    readonly timeline: string
+  }
 }
 
 export const defaults: Manifest = {
@@ -32,18 +35,16 @@ export function resolve() {
   if (!isManifest(manifest)) throw new Error(`Invalid drive manifest: ${file}`)
   validateEndpoint(manifest.endpoints.ui, "ui")
   validateEndpoint(manifest.endpoints.backend, "backend")
+  if (manifest.recording && !isAbsolute(manifest.recording.timeline)) {
+    throw new Error(`Invalid drive recording timeline path: ${manifest.recording.timeline}`)
+  }
   return manifest
 }
 
 function isManifest(value: unknown): value is Manifest {
-  if (typeof value !== "object" || value === null) return false
-  if (!("endpoints" in value) || typeof value.endpoints !== "object" || value.endpoints === null) return false
-  return (
-    "ui" in value.endpoints &&
-    typeof value.endpoints.ui === "string" &&
-    "backend" in value.endpoints &&
-    typeof value.endpoints.backend === "string"
-  )
+  if (typeof value !== "object" || value === null || !("endpoints" in value)) return false
+  if (typeof value.endpoints !== "object" || value.endpoints === null) return false
+  return "ui" in value.endpoints && "backend" in value.endpoints
 }
 
 function validateEndpoint(value: string, name: string) {

packages/simulation/src/protocol/index.ts

@@ -94,11 +94,11 @@ export namespace Frontend {
   export const Screenshot = Schema.String
   export type Screenshot = Schema.Schema.Type<typeof Screenshot>
 
-  export const StartRecord = Schema.Struct({ recording: Schema.Literal(true) })
-  export interface StartRecord extends Schema.Schema.Type<typeof StartRecord> {}
+  export const RecordingFinish = Schema.String
+  export type RecordingFinish = Schema.Schema.Type<typeof RecordingFinish>
 
-  export const EndRecord = Schema.String
-  export type EndRecord = Schema.Schema.Type<typeof EndRecord>
+  export const ArtifactParams = Schema.Struct({ path: Schema.optional(Schema.String) })
+  export interface ArtifactParams extends Schema.Schema.Type<typeof ArtifactParams> {}
 
   export const TypeParams = Schema.Struct({ text: Schema.String })
   export interface TypeParams extends Schema.Schema.Type<typeof TypeParams> {}
@@ -123,18 +123,16 @@ export namespace Frontend {
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.click"), params: ClickParams }),
     Schema.Struct({
       ...JsonRpc.RequestFields,
-      method: Schema.Literals([
-        "ui.enter",
-        "ui.screenshot",
-        "ui.state",
-        "ui.start-record",
-        "ui.end-record",
-      ]),
+      method: Schema.Literal("ui.screenshot"),
+      params: Schema.optional(ArtifactParams),
+    }),
+    Schema.Struct({
+      ...JsonRpc.RequestFields,
+      method: Schema.Literals(["ui.enter", "ui.state", "ui.recording.finish"]),
     }),
   ])
   export type Request = Schema.Schema.Type<typeof Request>
   export const decodeRequest = Schema.decodeUnknownSync(Request)
-
 }
 
 export namespace Backend {
@@ -189,7 +187,6 @@ export namespace Backend {
     matched: Schema.Boolean,
   })
   export interface NetworkLogEntry extends Schema.Schema.Type<typeof NetworkLogEntry> {}
-
 }
 
 export * as SimulationProtocol from "./index"

packages/simulation/src/recording.ts

@@ -0,0 +1,115 @@
+import { createWriteStream, type WriteStream } from "node:fs"
+import { mkdir } from "node:fs/promises"
+import { dirname } from "node:path"
+import { Writable } from "node:stream"
+import { finished } from "node:stream/promises"
+import { Schema } from "effect"
+
+export const Header = Schema.Struct({
+  type: Schema.Literal("header"),
+  version: Schema.Literal(1),
+  cols: Schema.Number,
+  rows: Schema.Number,
+  encoding: Schema.Literal("base64"),
+})
+export interface Header extends Schema.Schema.Type<typeof Header> {}
+
+export const Output = Schema.Struct({
+  type: Schema.Literal("output"),
+  at_ms: Schema.Number,
+  data: Schema.String,
+})
+export interface Output extends Schema.Schema.Type<typeof Output> {}
+
+export const Event = Schema.Union([Header, Output])
+export type Event = Schema.Schema.Type<typeof Event>
+
+export class Timeline extends Writable {
+  readonly isTTY = true
+  readonly path: string
+  readonly columns: number
+  readonly rows: number
+  private readonly output: WriteStream
+  private readonly started = performance.now()
+  private readonly timestamps: number[] = []
+  private done?: Promise<string>
+
+  private constructor(path: string, cols: number, rows: number, output: WriteStream) {
+    super()
+    this.path = path
+    this.columns = cols
+    this.rows = rows
+    this.output = output
+    // finish() reports stream failures; keep Writable from also throwing them process-wide.
+    this.on("error", () => {})
+    output.on("error", (error) => this.destroy(error))
+  }
+
+  static async create(path: string, cols: number, rows: number) {
+    await mkdir(dirname(path), { recursive: true })
+    const output = createWriteStream(path)
+    const timeline = new Timeline(path, cols, rows, output)
+    await new Promise<void>((resolve, reject) => {
+      output.write(
+        `${JSON.stringify({ type: "header", version: 1, cols, rows, encoding: "base64" } satisfies Header)}\n`,
+        (error) => (error ? reject(error) : resolve()),
+      )
+    })
+    return timeline
+  }
+
+  getColorDepth() {
+    return 24
+  }
+
+  override write(chunk: unknown, callback?: (error?: Error | null) => void): boolean
+  override write(chunk: unknown, encoding: BufferEncoding, callback?: (error?: Error | null) => void): boolean
+  override write(
+    chunk: unknown,
+    encoding?: BufferEncoding | ((error?: Error | null) => void),
+    callback?: (error?: Error | null) => void,
+  ) {
+    if (!this.writableEnded) {
+      this.timestamps.push(this.elapsed())
+      if (typeof encoding === "function") return super.write(chunk, encoding)
+      if (encoding === undefined) return super.write(chunk, callback)
+      return super.write(chunk, encoding, callback)
+    }
+    const done = typeof encoding === "function" ? encoding : callback
+    queueMicrotask(() => done?.(null))
+    return true
+  }
+
+  override _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) {
+    this.writeOutput(chunk, this.timestamps.shift() ?? this.elapsed(), callback)
+  }
+
+  override _final(callback: (error?: Error | null) => void) {
+    this.writeOutput(Buffer.alloc(0), this.elapsed(), (error) => {
+      if (error) return callback(error)
+      this.output.end(callback)
+    })
+  }
+
+  finish() {
+    if (this.done) return this.done
+    this.end()
+    this.done = finished(this).then(() => this.path)
+    return this.done
+  }
+
+  private elapsed() {
+    return Math.max(0, Math.round(performance.now() - this.started))
+  }
+
+  private writeOutput(data: Buffer, at_ms: number, callback: (error?: Error | null) => void) {
+    const event = {
+      type: "output",
+      at_ms,
+      data: data.toString("base64"),
+    } satisfies Output
+    this.output.write(`${JSON.stringify(event)}\n`, callback)
+  }
+}
+
+export * as SimulationRecording from "./recording"

packages/simulation/test/recording.test.ts

@@ -0,0 +1,55 @@
+import { expect, test } from "bun:test"
+import { mkdtemp, rm } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join } from "node:path"
+import { SimulationRenderer } from "../src/frontend/renderer"
+import { Timeline, type Event } from "../src/recording"
+
+test("streams ANSI chunks into a versioned JSONL timeline", async () => {
+  const directory = await mkdtemp(join(tmpdir(), "simulation-recording-"))
+  const path = join(directory, "nested", "timeline.jsonl")
+
+  try {
+    const timeline = await Timeline.create(path, 80, 24)
+    await new Promise<void>((resolve, reject) => {
+      timeline.write(Buffer.from("\u001b[2Jhello"), (error) => (error ? reject(error) : resolve()))
+    })
+    expect(await timeline.finish()).toBe(path)
+    await new Promise<void>((resolve) => timeline.write(Buffer.from("ignored"), () => resolve()))
+
+    const events = (await Bun.file(path).text())
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line) as Event)
+    expect(events[0]).toEqual({ type: "header", version: 1, cols: 80, rows: 24, encoding: "base64" })
+    expect(events[1]?.type).toBe("output")
+    if (events[1]?.type !== "output") throw new Error("Missing output event")
+    expect(Buffer.from(events[1].data, "base64").toString()).toBe("\u001b[2Jhello")
+    expect(events[1].at_ms).toBeGreaterThanOrEqual(0)
+    expect(events.at(-1)).toMatchObject({ type: "output", data: "" })
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+})
+
+test("captures native renderer output and finishes on destroy", async () => {
+  const directory = await mkdtemp(join(tmpdir(), "simulation-renderer-recording-"))
+  const path = join(directory, "timeline.jsonl")
+  const renderer = await SimulationRenderer.create({}, path)
+
+  try {
+    await SimulationRenderer.setupFor(renderer)?.renderOnce()
+    renderer.destroy()
+    expect(await SimulationRenderer.finish(renderer)).toBe(path)
+
+    const events = (await Bun.file(path).text())
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line) as Event)
+    expect(events.some((event) => event.type === "output")).toBe(true)
+  } finally {
+    if (!renderer.isDestroyed) renderer.destroy()
+    await SimulationRenderer.finish(renderer)
+    await rm(directory, { recursive: true, force: true })
+  }
+})