From c6156f171c1bb4ca77a1fe1fc5f28b1366502ac6 Mon Sep 17 00:00:00 2001 From: James Long Date: Wed, 8 Jul 2026 10:33:11 -0400 Subject: [PATCH] feat(simulation): stream UI recordings (#35909) --- packages/simulation/package.json | 3 +- packages/simulation/src/frontend/actions.ts | 68 ++--------- packages/simulation/src/frontend/renderer.ts | 38 +++++- packages/simulation/src/frontend/server.ts | 51 ++------ .../simulation/src/frontend/simulation.ts | 9 +- packages/simulation/src/manifest.ts | 19 +-- packages/simulation/src/protocol/index.ts | 23 ++-- packages/simulation/src/recording.ts | 115 ++++++++++++++++++ packages/simulation/test/recording.test.ts | 55 +++++++++ 9 files changed, 249 insertions(+), 132 deletions(-) create mode 100644 packages/simulation/src/recording.ts create mode 100644 packages/simulation/test/recording.test.ts diff --git a/packages/simulation/package.json b/packages/simulation/package.json index a98b6f3289..f9be3bfc33 100644 --- a/packages/simulation/package.json +++ b/packages/simulation/package.json @@ -10,7 +10,8 @@ "./backend/*": "./src/backend/*.ts", "./frontend": "./src/frontend/simulation.ts", "./frontend/*": "./src/frontend/*.ts", - "./protocol": "./src/protocol/index.ts" + "./protocol": "./src/protocol/index.ts", + "./recording": "./src/recording.ts" }, "scripts": { "typecheck": "tsgo --noEmit" diff --git a/packages/simulation/src/frontend/actions.ts b/packages/simulation/src/frontend/actions.ts index 4b03cb6939..068949459b 100644 --- a/packages/simulation/src/frontend/actions.ts +++ b/packages/simulation/src/frontend/actions.ts @@ -1,7 +1,7 @@ -import { mkdtemp } from "node:fs/promises" +import { mkdir, mkdtemp } from "node:fs/promises" import { tmpdir } from "node:os" -import { join } from "node:path" -import type { CapturedFrame, CliRenderer, Renderable } from "@opentui/core" +import { dirname, join } from "node:path" +import type { CliRenderer, Renderable } from "@opentui/core" import { createMockKeys, createMockMouse, type MockInput, type MockMouse } from "@opentui/core/testing" import type { SimulationProtocol } from "../protocol" import { SimulationRenderer } from "./renderer" @@ -104,67 +104,15 @@ export function state(harness: Harness) { } } -export async function screenshot(harness: Harness) { +export async function screenshot(harness: Harness, output?: string) { await harness.renderOnce() const image = SimulationPng.screenshot(harness.renderer) - const path = join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png") + const path = output ?? join(await mkdtemp(join(tmpdir(), "opencode-drive-")), "screenshot.png") + if (output) await mkdir(dirname(output), { recursive: true }) await Bun.write(path, image.data) return path } -export function frame(harness: Harness): CapturedFrame { - const buffer = harness.renderer.currentRenderBuffer - return { - cols: buffer.width, - rows: buffer.height, - cursor: [0, 0], - lines: buffer.getSpanLines().map((line) => ({ - spans: line.spans.map((span) => ({ - text: span.text, - fg: span.fg, - bg: span.bg, - attributes: span.attributes, - width: span.width, - })), - })), - } -} - -export async function video(frames: CapturedFrame[]) { - const directory = await mkdtemp(join(tmpdir(), "opencode-drive-recording-")) - await Promise.all( - frames.map((frame, index) => - Bun.write( - join(directory, `frame-${index.toString().padStart(6, "0")}.png`), - SimulationPng.screenshotFrame(frame).data, - ), - ), - ) - const path = join(directory, "recording.mp4") - const process = Bun.spawn( - [ - "ffmpeg", - "-loglevel", - "error", - "-framerate", - "10", - "-i", - join(directory, "frame-%06d.png"), - "-c:v", - "libx264", - "-pix_fmt", - "yuv420p", - "-movflags", - "+faststart", - "-y", - path, - ], - { stderr: "pipe" }, - ) - if ((await process.exited) !== 0) throw new Error(`ffmpeg failed: ${await new Response(process.stderr).text()}`) - return path -} - export async function execute(harness: Harness, action: Action) { switch (action.type) { case "ui.type": @@ -180,7 +128,9 @@ export async function execute(harness: Harness, action: Action) { harness.mockInput.pressArrow(action.direction) break case "ui.focus": - all(harness.renderer.root).find((item) => item.num === action.target)?.focus() + all(harness.renderer.root) + .find((item) => item.num === action.target) + ?.focus() break case "ui.click": await harness.mockMouse.click(action.x, action.y) diff --git a/packages/simulation/src/frontend/renderer.ts b/packages/simulation/src/frontend/renderer.ts index e973d57a98..43947d9ea6 100644 --- a/packages/simulation/src/frontend/renderer.ts +++ b/packages/simulation/src/frontend/renderer.ts @@ -1,21 +1,43 @@ import type { CliRenderer, CliRendererConfig } from "@opentui/core" import { createTestRenderer, type TestRendererSetup } from "@opentui/core/testing" +import { Timeline } from "../recording" const setups = new WeakMap() +const recordings = new WeakMap() /** - * Creates the headless simulation renderer: a real CliRenderer backed by an - * in-memory screen buffer instead of a terminal. The TestRendererSetup is - * kept module-side (keyed by renderer) so the harness can use the supported - * testing APIs without app code carrying it around. + * Creates a headless renderer with optional recording: a real CliRenderer + * backed by an in-memory screen buffer. The TestRendererSetup is kept + * module-side so the harness can use supported testing APIs without app + * code carrying it around. */ -export async function create(options: CliRendererConfig): Promise { +export async function create(options: CliRendererConfig, path?: string): Promise { + if (!path) { + const setup = await createTestRenderer({ + ...options, + width: 100, + height: 40, + }) + setups.set(setup.renderer, setup) + return setup.renderer + } + const recording = await Timeline.create(path, 100, 40) const setup = await createTestRenderer({ ...options, width: 100, height: 40, + stdout: recording as unknown as NodeJS.WriteStream, + bufferedOutput: "stdout", + onDestroy: () => { + void recording.finish().catch((error) => process.stderr.write(`Failed to finish UI recording: ${error}\n`)) + options.onDestroy?.() + }, + }).catch(async (error) => { + await recording.finish().catch(() => undefined) + throw error }) setups.set(setup.renderer, setup) + recordings.set(setup.renderer, recording) return setup.renderer } @@ -23,4 +45,10 @@ export function setupFor(renderer: CliRenderer): TestRendererSetup | undefined { return setups.get(renderer) } +export function finish(renderer: CliRenderer) { + const recording = recordings.get(renderer) + if (!recording) throw new Error("UI recording is not available") + return recording.finish() +} + export * as SimulationRenderer from "./renderer" diff --git a/packages/simulation/src/frontend/server.ts b/packages/simulation/src/frontend/server.ts index 4edd687887..e3149c39f8 100644 --- a/packages/simulation/src/frontend/server.ts +++ b/packages/simulation/src/frontend/server.ts @@ -1,4 +1,3 @@ -import type { CapturedFrame } from "@opentui/core" import { SimulationProtocol } from "../protocol" import { SimulationActions, type Harness } from "./actions" @@ -11,50 +10,20 @@ function parseRequest(input: string | Buffer) { return SimulationProtocol.Frontend.decodeRequest(JSON.parse(typeof input === "string" ? input : input.toString())) } -interface Recording { - readonly frames: CapturedFrame[] - readonly timer: ReturnType - pending: Promise -} - async function handle( harness: Harness, request: SimulationProtocol.Frontend.Request, - recording: { current?: Recording }, - headless: boolean, + finishRecording?: () => Promise, ) { switch (request.method) { case "ui.screenshot": - return SimulationActions.screenshot(harness) + return SimulationActions.screenshot(harness, request.params?.path) case "ui.state": { return SimulationActions.state(harness) } - case "ui.start-record": { - if (recording.current) throw new Error("UI recording is already active") - const frames = [SimulationActions.frame(harness)] - const current: Recording = { - frames, - timer: setInterval(() => { - current.pending = current.pending.then(async () => { - if (headless) await harness.renderOnce() - frames.push(SimulationActions.frame(harness)) - }) - }, 100), - pending: Promise.resolve(), - } - recording.current = current - return { recording: true } - } - case "ui.end-record": { - if (!recording.current) throw new Error("UI recording is not active") - const current = recording.current - clearInterval(current.timer) - await current.pending - if (headless) await harness.renderOnce() - current.frames.push(SimulationActions.frame(harness)) - recording.current = undefined - return SimulationActions.video(current.frames) - } + case "ui.recording.finish": + if (!finishRecording) throw new Error("UI recording is not available") + return finishRecording() case "ui.type": return SimulationActions.execute(harness, { type: "ui.type", text: request.params.text }) case "ui.enter": @@ -79,14 +48,13 @@ async function handle( } } -export function start(harness: Harness, endpoint: string, headless: boolean): Server { +export function start(harness: Harness, endpoint: string, finishRecording?: () => Promise): Server { const url = new URL(endpoint) - const recording: { current?: Recording } = {} - const server = Bun.serve<{ readonly drive: true; readonly headless: boolean }>({ + const server = Bun.serve<{ readonly drive: true }>({ hostname: url.hostname, port: Number(url.port), fetch(request, server) { - if (server.upgrade(request, { data: { drive: true, headless } })) return undefined + if (server.upgrade(request, { data: { drive: true } })) return undefined return new Response("opencode drive ui websocket", { status: 426 }) }, websocket: { @@ -94,7 +62,7 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se let request: SimulationProtocol.Frontend.Request | undefined try { request = parseRequest(message) - const result = await handle(harness, request, recording, headless) + const result = await handle(harness, request, finishRecording) const next = SimulationProtocol.JsonRpc.success(request.id, result) if (next) socket.send(JSON.stringify(next)) } catch (error) { @@ -106,7 +74,6 @@ export function start(harness: Harness, endpoint: string, headless: boolean): Se return { url: endpoint, stop: () => { - if (recording.current) clearInterval(recording.current.timer) server.stop(true) }, } diff --git a/packages/simulation/src/frontend/simulation.ts b/packages/simulation/src/frontend/simulation.ts index c2a56ad8c1..2a08d2dbef 100644 --- a/packages/simulation/src/frontend/simulation.ts +++ b/packages/simulation/src/frontend/simulation.ts @@ -14,11 +14,14 @@ import { SimulationServer } from "./server" */ export async function create(options: CliRendererConfig): Promise { const headless = process.env.OPENCODE_DRIVE_RENDERER === "headless" - const renderer = headless ? await SimulationRenderer.create(options) : await createCliRenderer(options) + const manifest = DriveManifest.resolve() + const renderer = headless + ? await SimulationRenderer.create(options, manifest.recording?.timeline) + : await createCliRenderer(options) const server = SimulationServer.start( SimulationActions.createHarness(renderer), - DriveManifest.resolve().endpoints.ui, - headless, + manifest.endpoints.ui, + headless && manifest.recording ? () => SimulationRenderer.finish(renderer) : undefined, ) process.stderr.write(`opencode drive ui websocket: ${server.url}\n`) renderer.once("destroy", () => server.stop()) diff --git a/packages/simulation/src/manifest.ts b/packages/simulation/src/manifest.ts index 0ef4530e48..5544d23b70 100644 --- a/packages/simulation/src/manifest.ts +++ b/packages/simulation/src/manifest.ts @@ -1,12 +1,15 @@ import { existsSync, readFileSync } from "node:fs" import { homedir } from "node:os" -import { join } from "node:path" +import { isAbsolute, join } from "node:path" export interface Manifest { readonly endpoints: { readonly ui: string readonly backend: string } + readonly recording?: { + readonly timeline: string + } } export const defaults: Manifest = { @@ -32,18 +35,16 @@ export function resolve() { if (!isManifest(manifest)) throw new Error(`Invalid drive manifest: ${file}`) validateEndpoint(manifest.endpoints.ui, "ui") validateEndpoint(manifest.endpoints.backend, "backend") + if (manifest.recording && !isAbsolute(manifest.recording.timeline)) { + throw new Error(`Invalid drive recording timeline path: ${manifest.recording.timeline}`) + } return manifest } function isManifest(value: unknown): value is Manifest { - if (typeof value !== "object" || value === null) return false - if (!("endpoints" in value) || typeof value.endpoints !== "object" || value.endpoints === null) return false - return ( - "ui" in value.endpoints && - typeof value.endpoints.ui === "string" && - "backend" in value.endpoints && - typeof value.endpoints.backend === "string" - ) + if (typeof value !== "object" || value === null || !("endpoints" in value)) return false + if (typeof value.endpoints !== "object" || value.endpoints === null) return false + return "ui" in value.endpoints && "backend" in value.endpoints } function validateEndpoint(value: string, name: string) { diff --git a/packages/simulation/src/protocol/index.ts b/packages/simulation/src/protocol/index.ts index 73c43f4119..eda31c30bf 100644 --- a/packages/simulation/src/protocol/index.ts +++ b/packages/simulation/src/protocol/index.ts @@ -94,11 +94,11 @@ export namespace Frontend { export const Screenshot = Schema.String export type Screenshot = Schema.Schema.Type - export const StartRecord = Schema.Struct({ recording: Schema.Literal(true) }) - export interface StartRecord extends Schema.Schema.Type {} + export const RecordingFinish = Schema.String + export type RecordingFinish = Schema.Schema.Type - export const EndRecord = Schema.String - export type EndRecord = Schema.Schema.Type + export const ArtifactParams = Schema.Struct({ path: Schema.optional(Schema.String) }) + export interface ArtifactParams extends Schema.Schema.Type {} export const TypeParams = Schema.Struct({ text: Schema.String }) export interface TypeParams extends Schema.Schema.Type {} @@ -123,18 +123,16 @@ export namespace Frontend { Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.click"), params: ClickParams }), Schema.Struct({ ...JsonRpc.RequestFields, - method: Schema.Literals([ - "ui.enter", - "ui.screenshot", - "ui.state", - "ui.start-record", - "ui.end-record", - ]), + method: Schema.Literal("ui.screenshot"), + params: Schema.optional(ArtifactParams), + }), + Schema.Struct({ + ...JsonRpc.RequestFields, + method: Schema.Literals(["ui.enter", "ui.state", "ui.recording.finish"]), }), ]) export type Request = Schema.Schema.Type export const decodeRequest = Schema.decodeUnknownSync(Request) - } export namespace Backend { @@ -189,7 +187,6 @@ export namespace Backend { matched: Schema.Boolean, }) export interface NetworkLogEntry extends Schema.Schema.Type {} - } export * as SimulationProtocol from "./index" diff --git a/packages/simulation/src/recording.ts b/packages/simulation/src/recording.ts new file mode 100644 index 0000000000..6d0b3a6bd5 --- /dev/null +++ b/packages/simulation/src/recording.ts @@ -0,0 +1,115 @@ +import { createWriteStream, type WriteStream } from "node:fs" +import { mkdir } from "node:fs/promises" +import { dirname } from "node:path" +import { Writable } from "node:stream" +import { finished } from "node:stream/promises" +import { Schema } from "effect" + +export const Header = Schema.Struct({ + type: Schema.Literal("header"), + version: Schema.Literal(1), + cols: Schema.Number, + rows: Schema.Number, + encoding: Schema.Literal("base64"), +}) +export interface Header extends Schema.Schema.Type {} + +export const Output = Schema.Struct({ + type: Schema.Literal("output"), + at_ms: Schema.Number, + data: Schema.String, +}) +export interface Output extends Schema.Schema.Type {} + +export const Event = Schema.Union([Header, Output]) +export type Event = Schema.Schema.Type + +export class Timeline extends Writable { + readonly isTTY = true + readonly path: string + readonly columns: number + readonly rows: number + private readonly output: WriteStream + private readonly started = performance.now() + private readonly timestamps: number[] = [] + private done?: Promise + + private constructor(path: string, cols: number, rows: number, output: WriteStream) { + super() + this.path = path + this.columns = cols + this.rows = rows + this.output = output + // finish() reports stream failures; keep Writable from also throwing them process-wide. + this.on("error", () => {}) + output.on("error", (error) => this.destroy(error)) + } + + static async create(path: string, cols: number, rows: number) { + await mkdir(dirname(path), { recursive: true }) + const output = createWriteStream(path) + const timeline = new Timeline(path, cols, rows, output) + await new Promise((resolve, reject) => { + output.write( + `${JSON.stringify({ type: "header", version: 1, cols, rows, encoding: "base64" } satisfies Header)}\n`, + (error) => (error ? reject(error) : resolve()), + ) + }) + return timeline + } + + getColorDepth() { + return 24 + } + + override write(chunk: unknown, callback?: (error?: Error | null) => void): boolean + override write(chunk: unknown, encoding: BufferEncoding, callback?: (error?: Error | null) => void): boolean + override write( + chunk: unknown, + encoding?: BufferEncoding | ((error?: Error | null) => void), + callback?: (error?: Error | null) => void, + ) { + if (!this.writableEnded) { + this.timestamps.push(this.elapsed()) + if (typeof encoding === "function") return super.write(chunk, encoding) + if (encoding === undefined) return super.write(chunk, callback) + return super.write(chunk, encoding, callback) + } + const done = typeof encoding === "function" ? encoding : callback + queueMicrotask(() => done?.(null)) + return true + } + + override _write(chunk: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void) { + this.writeOutput(chunk, this.timestamps.shift() ?? this.elapsed(), callback) + } + + override _final(callback: (error?: Error | null) => void) { + this.writeOutput(Buffer.alloc(0), this.elapsed(), (error) => { + if (error) return callback(error) + this.output.end(callback) + }) + } + + finish() { + if (this.done) return this.done + this.end() + this.done = finished(this).then(() => this.path) + return this.done + } + + private elapsed() { + return Math.max(0, Math.round(performance.now() - this.started)) + } + + private writeOutput(data: Buffer, at_ms: number, callback: (error?: Error | null) => void) { + const event = { + type: "output", + at_ms, + data: data.toString("base64"), + } satisfies Output + this.output.write(`${JSON.stringify(event)}\n`, callback) + } +} + +export * as SimulationRecording from "./recording" diff --git a/packages/simulation/test/recording.test.ts b/packages/simulation/test/recording.test.ts new file mode 100644 index 0000000000..d87453bb79 --- /dev/null +++ b/packages/simulation/test/recording.test.ts @@ -0,0 +1,55 @@ +import { expect, test } from "bun:test" +import { mkdtemp, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { SimulationRenderer } from "../src/frontend/renderer" +import { Timeline, type Event } from "../src/recording" + +test("streams ANSI chunks into a versioned JSONL timeline", async () => { + const directory = await mkdtemp(join(tmpdir(), "simulation-recording-")) + const path = join(directory, "nested", "timeline.jsonl") + + try { + const timeline = await Timeline.create(path, 80, 24) + await new Promise((resolve, reject) => { + timeline.write(Buffer.from("\u001b[2Jhello"), (error) => (error ? reject(error) : resolve())) + }) + expect(await timeline.finish()).toBe(path) + await new Promise((resolve) => timeline.write(Buffer.from("ignored"), () => resolve())) + + const events = (await Bun.file(path).text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Event) + expect(events[0]).toEqual({ type: "header", version: 1, cols: 80, rows: 24, encoding: "base64" }) + expect(events[1]?.type).toBe("output") + if (events[1]?.type !== "output") throw new Error("Missing output event") + expect(Buffer.from(events[1].data, "base64").toString()).toBe("\u001b[2Jhello") + expect(events[1].at_ms).toBeGreaterThanOrEqual(0) + expect(events.at(-1)).toMatchObject({ type: "output", data: "" }) + } finally { + await rm(directory, { recursive: true, force: true }) + } +}) + +test("captures native renderer output and finishes on destroy", async () => { + const directory = await mkdtemp(join(tmpdir(), "simulation-renderer-recording-")) + const path = join(directory, "timeline.jsonl") + const renderer = await SimulationRenderer.create({}, path) + + try { + await SimulationRenderer.setupFor(renderer)?.renderOnce() + renderer.destroy() + expect(await SimulationRenderer.finish(renderer)).toBe(path) + + const events = (await Bun.file(path).text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Event) + expect(events.some((event) => event.type === "output")).toBe(true) + } finally { + if (!renderer.isDestroyed) renderer.destroy() + await SimulationRenderer.finish(renderer) + await rm(directory, { recursive: true, force: true }) + } +})