feat(simulation): expose semantic UI snapshots (#37802)

packages/simulation/src/frontend/actions.ts

@@ -9,9 +9,10 @@ import {
   type MockInput,
   type MockMouse,
 } from "@opentui/core/testing"
-import { Config, Effect, FileSystem } from "effect"
-import type { SimulationProtocol } from "../protocol"
+import { Config, Effect, FileSystem, Schema } from "effect"
+import { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
+import { SimulationSemantics } from "./semantics"
 
 export type Action = SimulationProtocol.Frontend.Action
 export type Element = SimulationProtocol.Frontend.Element
@@ -60,7 +61,8 @@ function hit(renderer: CliRenderer, renderable: Renderable) {
   if (renderable.width <= 0 || renderable.height <= 0) return false
   const x = Math.floor(renderable.screenX + renderable.width / 2)
   const y = Math.floor(renderable.screenY + renderable.height / 2)
-  return renderer.hitTest(x, y) === renderable.num
+  const target = renderer.hitTest(x, y)
+  return all(renderable).some((item) => item.num === target)
 }
 
 /**
@@ -122,6 +124,25 @@ export function state(harness: Harness) {
   }
 }
 
+export function snapshot(harness: Harness): SimulationProtocol.Frontend.SemanticSnapshot {
+  const ids = new Set<string>()
+  const visit = (renderable: Renderable, parent?: string): SimulationProtocol.Frontend.SemanticNode[] => {
+    if (!renderable.visible || renderable.isDestroyed) return []
+    const definition = SimulationSemantics.read(renderable)?.()
+    if (definition && ids.has(renderable.id)) throw new Error(`duplicate semantic UI id: ${renderable.id}`)
+    if (definition) ids.add(renderable.id)
+    const node = definition
+      ? [{ id: renderable.id, ...definition, ...(parent === undefined ? {} : { parent }), element: renderable.num }]
+      : []
+    const ancestor = definition ? renderable.id : parent
+    return [...node, ...children(renderable).flatMap((child) => visit(child, ancestor))]
+  }
+  return Schema.decodeUnknownSync(SimulationProtocol.Frontend.SemanticSnapshot)({
+    format: "opencode-ui-snapshot-v1",
+    nodes: visit(harness.renderer.root),
+  })
+}
+
 export function matches(harness: Pick<Harness, "screen">, text: string) {
   return harness.screen().includes(text)
 }
@@ -183,9 +204,22 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness
         .find((item) => item.num === action.target)
         ?.focus()
       break
-    case "ui.click":
-      yield* Effect.tryPromise(() => harness.mockMouse.click(action.x, action.y))
+    case "ui.click": {
+      const target = all(harness.renderer.root).find((item) => item.num === action.target)
+      if (!target || !target.visible || target.isDestroyed)
+        return yield* Effect.fail(new Error(`click target is stale or unavailable: ${action.target}`))
+      if (
+        !Number.isFinite(action.x) ||
+        action.x < 0 ||
+        action.x >= target.width ||
+        !Number.isFinite(action.y) ||
+        action.y < 0 ||
+        action.y >= target.height
+      )
+        return yield* Effect.fail(new Error("click position must be within the target element"))
+      yield* Effect.tryPromise(() => harness.mockMouse.click(target.screenX + action.x, target.screenY + action.y))
       break
+    }
     case "ui.resize":
       if (
         !Number.isSafeInteger(action.cols) ||

packages/simulation/src/frontend/semantics.ts

@@ -0,0 +1,20 @@
+import type { Renderable } from "@opentui/core"
+import type { SimulationProtocol } from "../protocol"
+
+// Semantic renderables set an explicit stable OpenTUI id so ui.state and
+// ui.snapshot expose the same identity. Hierarchy and element handles come
+// from the live render tree.
+export type Definition = Omit<SimulationProtocol.Frontend.SemanticNode, "id" | "element" | "parent">
+
+const key = Symbol.for("opencode.simulation.semantics")
+
+const bind = (definition: () => Definition) => (renderable: Renderable) => {
+  Object.defineProperty(renderable, key, { value: definition, configurable: true })
+}
+
+export const read = (renderable: Renderable) => {
+  const definition: unknown = Reflect.get(renderable, key)
+  return typeof definition === "function" ? (definition as () => Definition) : undefined
+}
+
+export const SimulationSemantics = { bind, read }

packages/simulation/src/frontend/server.ts

@@ -22,6 +22,8 @@ function handle(harness: Harness, request: SimulationProtocol.Frontend.Request)
       return SimulationActions.screenshot(harness, request.params?.name)
     case "ui.state":
       return Effect.sync(() => SimulationActions.state(harness))
+    case "ui.snapshot":
+      return Effect.sync(() => SimulationActions.snapshot(harness))
     case "ui.matches":
       return Effect.sync(() => SimulationActions.matches(harness, request.params.text))
     case "ui.recording.finish":

packages/simulation/src/protocol/index.ts

@@ -67,9 +67,10 @@ export namespace Handshake {
   export const Params = Schema.Struct({
     client: Identity,
     expectedRole: EndpointRole,
-    offeredVersions: Schema.Array(
-      Schema.Int.check(Schema.isGreaterThan(0)),
-    ).check(Schema.isMinLength(1), Schema.isUnique()),
+    offeredVersions: Schema.Array(Schema.Int.check(Schema.isGreaterThan(0))).check(
+      Schema.isMinLength(1),
+      Schema.isUnique(),
+    ),
     requiredCapabilities: Schema.Array(Capability).check(Schema.isUnique()),
     optionalCapabilities: Schema.Array(Capability).check(Schema.isUnique()),
   })
@@ -174,6 +175,7 @@ export namespace Frontend {
     "ui.matches",
     "ui.screenshot",
     "ui.state",
+    "ui.snapshot",
     "ui.capture",
     "ui.recording.finish",
   ] as const satisfies ReadonlyArray<Handshake.Capability>
@@ -221,6 +223,46 @@ export namespace Frontend {
   })
   export interface State extends Schema.Schema.Type<typeof State> {}
 
+  export const SemanticNode = Schema.Struct({
+    id: Schema.NonEmptyString,
+    instance: Schema.optionalKey(Schema.NonEmptyString),
+    parent: Schema.optionalKey(Schema.NonEmptyString),
+    role: Schema.NonEmptyString,
+    label: Schema.optionalKey(Schema.NonEmptyString),
+    element: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)),
+    focused: Schema.optionalKey(Schema.Boolean),
+    selected: Schema.optionalKey(Schema.Boolean),
+    expanded: Schema.optionalKey(Schema.Boolean),
+    disabled: Schema.optionalKey(Schema.Boolean),
+  })
+  export interface SemanticNode extends Schema.Schema.Type<typeof SemanticNode> {}
+
+  export const SemanticSnapshot = Schema.Struct({
+    format: Schema.Literal("opencode-ui-snapshot-v1"),
+    nodes: Schema.Array(SemanticNode).check(
+      Schema.makeFilter((nodes) => {
+        const ids = new Set(nodes.map((node) => node.id))
+        if (ids.size !== nodes.length) return "semantic node ids must be unique"
+        if (new Set(nodes.map((node) => node.element)).size !== nodes.length)
+          return "semantic node elements must be unique"
+        if (nodes.some((node) => node.parent !== undefined && !ids.has(node.parent)))
+          return "semantic node parents must reference another node"
+        const parents = new Map(nodes.map((node) => [node.id, node.parent]))
+        for (const node of nodes) {
+          const visited = new Set<string>()
+          let current: string | undefined = node.id
+          while (current !== undefined) {
+            if (visited.has(current)) return "semantic node hierarchy must be acyclic"
+            visited.add(current)
+            current = parents.get(current)
+          }
+        }
+        return undefined
+      }),
+    ),
+  })
+  export interface SemanticSnapshot extends Schema.Schema.Type<typeof SemanticSnapshot> {}
+
   export const Screenshot = Schema.String
   export type Screenshot = Schema.Schema.Type<typeof Screenshot>
 
@@ -293,7 +335,7 @@ export namespace Frontend {
     }),
     Schema.Struct({
       ...JsonRpc.RequestFields,
-      method: Schema.Literals(["ui.enter", "ui.state", "ui.recording.finish"]),
+      method: Schema.Literals(["ui.enter", "ui.state", "ui.snapshot", "ui.recording.finish"]),
     }),
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.capture") }),
   ])

packages/simulation/test/actions.test.ts

@@ -1,6 +1,9 @@
 import { expect, test } from "bun:test"
+import { BoxRenderable, TextRenderable } from "@opentui/core"
 import { Effect } from "effect"
-import { execute, type Harness, matches } from "../src/frontend/actions"
+import { createHarness, execute, type Harness, matches, snapshot, state } from "../src/frontend/actions"
+import { SimulationRenderer } from "../src/frontend/renderer"
+import { SimulationSemantics } from "../src/frontend/semantics"
 
 test("matches literal screen text", () => {
   const harness = { screen: () => "OpenCode [ready].*" }
@@ -39,3 +42,144 @@ test("normalizes named keys for OpenTUI", async () => {
     ["x", undefined],
   ])
 })
+
+test("clicks a target at relative coordinates through descendant text", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        let clicks = 0
+        const button = new BoxRenderable(renderer, {
+          id: "permission.action.once",
+          width: 12,
+          height: 1,
+          onMouseUp: () => clicks++,
+        })
+        button.add(new TextRenderable(renderer, { content: "Allow once" }))
+        renderer.root.add(button)
+        const harness = createHarness(renderer)
+        yield* Effect.promise(() => harness.renderOnce())
+
+        expect(state(harness).elements).toContainEqual(expect.objectContaining({ id: button.id, clickable: true }))
+        yield* execute(harness, { type: "ui.click", target: button.num, x: 1, y: 0 })
+        expect(clicks).toBe(1)
+
+        renderer.root.remove(button)
+        const error = yield* execute(harness, { type: "ui.click", target: button.num, x: 1, y: 0 }).pipe(Effect.flip)
+        expect(error.message).toContain("click target is stale or unavailable")
+      }),
+    ),
+  )
+})
+
+test("snapshots lazy semantic hierarchy and interaction state", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        let selected = "once"
+        const dialog = new BoxRenderable(renderer, { id: "session.permission" })
+        const actions = new BoxRenderable(renderer, { id: "session.permission.actions" })
+        const once = new BoxRenderable(renderer, { id: "session.permission.action.once" })
+        SimulationSemantics.bind(() => ({
+          instance: "permission-1",
+          role: "dialog",
+          label: "Permission required",
+          expanded: false,
+        }))(dialog)
+        SimulationSemantics.bind(() => ({
+          instance: "permission-1",
+          role: "listbox",
+          label: "Permission choices",
+        }))(actions)
+        SimulationSemantics.bind(() => ({
+          instance: "permission-1",
+          role: "option",
+          label: "Allow once",
+          focused: selected === "once",
+          selected: selected === "once",
+          disabled: false,
+        }))(once)
+        renderer.root.add(dialog)
+        dialog.add(actions)
+        actions.add(once)
+
+        expect(snapshot(createHarness(renderer))).toEqual({
+          format: "opencode-ui-snapshot-v1",
+          nodes: [
+            {
+              id: "session.permission",
+              instance: "permission-1",
+              role: "dialog",
+              label: "Permission required",
+              element: dialog.num,
+              expanded: false,
+            },
+            {
+              id: "session.permission.actions",
+              instance: "permission-1",
+              parent: "session.permission",
+              role: "listbox",
+              label: "Permission choices",
+              element: actions.num,
+            },
+            {
+              id: "session.permission.action.once",
+              instance: "permission-1",
+              parent: "session.permission.actions",
+              role: "option",
+              label: "Allow once",
+              element: once.num,
+              focused: true,
+              selected: true,
+              disabled: false,
+            },
+          ],
+        })
+
+        selected = "reject"
+        expect(snapshot(createHarness(renderer)).nodes.at(-1)).toMatchObject({
+          id: "session.permission.action.once",
+          focused: false,
+          selected: false,
+        })
+        dialog.visible = false
+        expect(snapshot(createHarness(renderer)).nodes).toEqual([])
+      }),
+    ),
+  )
+})
+
+test("rejects duplicate semantic identities", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        const first = new BoxRenderable(renderer, { id: "duplicate" })
+        const second = new BoxRenderable(renderer, { id: "duplicate" })
+        const definition = () => ({ role: "option" })
+        SimulationSemantics.bind(definition)(first)
+        SimulationSemantics.bind(definition)(second)
+        renderer.root.add(first)
+        renderer.root.add(second)
+
+        expect(() => snapshot(createHarness(renderer))).toThrow("duplicate semantic UI id: duplicate")
+      }),
+    ),
+  )
+})
+
+test("validates lazy semantic definitions before returning them", async () => {
+  await Effect.runPromise(
+    Effect.scoped(
+      Effect.gen(function* () {
+        const renderer = yield* SimulationRenderer.create({})
+        const invalid = new BoxRenderable(renderer, { id: "invalid" })
+        SimulationSemantics.bind(() => ({ role: "" }))(invalid)
+        renderer.root.add(invalid)
+
+        expect(() => snapshot(createHarness(renderer))).toThrow()
+      }),
+    ),
+  )
+})

packages/simulation/test/frontend-server.test.ts

@@ -39,7 +39,7 @@ test("scopes the frontend control server and reports malformed JSON", async () =
             protocolVersion: 1,
             role: "ui",
             server: { name: "opencode", version: expect.any(String) },
-            capabilities: expect.arrayContaining(["ui.state", "ui.capture"]),
+            capabilities: expect.arrayContaining(["ui.state", "ui.snapshot", "ui.capture"]),
           },
         })
 
@@ -60,6 +60,13 @@ test("scopes the frontend control server and reports malformed JSON", async () =
           },
         })
 
+        socket.send(JSON.stringify({ jsonrpc: "2.0", id: 3, method: "ui.snapshot" }))
+        expect(yield* Queue.take(messages)).toEqual({
+          jsonrpc: "2.0",
+          id: 3,
+          result: { format: "opencode-ui-snapshot-v1", nodes: [] },
+        })
+
         socket.send("{")
         expect(yield* Queue.take(messages)).toMatchObject({
           id: null,

packages/simulation/test/protocol.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test"
-import { Effect } from "effect"
+import { Effect, Schema } from "effect"
 import { Backend, Frontend, Handshake } from "../src/protocol"
 
 test("decodes ui.matches text params", () => {
@@ -21,6 +21,53 @@ test("decodes ui.matches text params", () => {
   ).toThrow()
 })
 
+test("decodes semantic UI snapshots", () => {
+  expect(
+    Frontend.decodeRequest({
+      jsonrpc: "2.0",
+      id: 1,
+      method: "ui.snapshot",
+    }),
+  ).toMatchObject({ method: "ui.snapshot" })
+  const decode = Schema.decodeUnknownSync(Frontend.SemanticSnapshot)
+  expect(
+    decode({
+      format: "opencode-ui-snapshot-v1",
+      nodes: [
+        {
+          id: "session.permission",
+          role: "dialog",
+          label: "Permission required",
+          element: 1,
+          expanded: false,
+        },
+      ],
+    }),
+  ).toMatchObject({ nodes: [{ role: "dialog", expanded: false }] })
+  expect(() =>
+    decode({
+      format: "opencode-ui-snapshot-v1",
+      nodes: [{ id: "", role: "dialog", element: 0 }],
+    }),
+  ).toThrow()
+  for (const nodes of [
+    [
+      { id: "duplicate", role: "dialog", element: 1 },
+      { id: "duplicate", role: "option", element: 2 },
+    ],
+    [
+      { id: "first", role: "dialog", element: 1 },
+      { id: "second", role: "option", element: 1 },
+    ],
+    [{ id: "orphan", parent: "missing", role: "option", element: 1 }],
+    [
+      { id: "first", parent: "second", role: "dialog", element: 1 },
+      { id: "second", parent: "first", role: "option", element: 2 },
+    ],
+  ])
+    expect(() => decode({ format: "opencode-ui-snapshot-v1", nodes })).toThrow()
+})
+
 const params: Handshake.Params = {
   client: { name: "opencode-drive", version: "test" },
   expectedRole: "ui",