feat(simulation): expose semantic UI snapshots (#37802)

packages/simulation/src/frontend/actions.ts

@@ -9,9 +9,10 @@ import {
   type MockInput,
   type MockMouse,
 } from "@opentui/core/testing"
-import { Config, Effect, FileSystem } from "effect"
-import type { SimulationProtocol } from "../protocol"
+import { Config, Effect, FileSystem, Schema } from "effect"
+import { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
+import { SimulationSemantics } from "./semantics"
 
 export type Action = SimulationProtocol.Frontend.Action
 export type Element = SimulationProtocol.Frontend.Element
@@ -60,7 +61,8 @@ function hit(renderer: CliRenderer, renderable: Renderable) {
   if (renderable.width <= 0 || renderable.height <= 0) return false
   const x = Math.floor(renderable.screenX + renderable.width / 2)
   const y = Math.floor(renderable.screenY + renderable.height / 2)
-  return renderer.hitTest(x, y) === renderable.num
+  const target = renderer.hitTest(x, y)
+  return all(renderable).some((item) => item.num === target)
 }
 
 /**
@@ -122,6 +124,25 @@ export function state(harness: Harness) {
   }
 }
 
+export function snapshot(harness: Harness): SimulationProtocol.Frontend.SemanticSnapshot {
+  const ids = new Set<string>()
+  const visit = (renderable: Renderable, parent?: string): SimulationProtocol.Frontend.SemanticNode[] => {
+    if (!renderable.visible || renderable.isDestroyed) return []
+    const definition = SimulationSemantics.read(renderable)?.()
+    if (definition && ids.has(renderable.id)) throw new Error(`duplicate semantic UI id: ${renderable.id}`)
+    if (definition) ids.add(renderable.id)
+    const node = definition
+      ? [{ id: renderable.id, ...definition, ...(parent === undefined ? {} : { parent }), element: renderable.num }]
+      : []
+    const ancestor = definition ? renderable.id : parent
+    return [...node, ...children(renderable).flatMap((child) => visit(child, ancestor))]
+  }
+  return Schema.decodeUnknownSync(SimulationProtocol.Frontend.SemanticSnapshot)({
+    format: "opencode-ui-snapshot-v1",
+    nodes: visit(harness.renderer.root),
+  })
+}
+
 export function matches(harness: Pick<Harness, "screen">, text: string) {
   return harness.screen().includes(text)
 }
@@ -183,9 +204,22 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness
         .find((item) => item.num === action.target)
         ?.focus()
       break
-    case "ui.click":
-      yield* Effect.tryPromise(() => harness.mockMouse.click(action.x, action.y))
+    case "ui.click": {
+      const target = all(harness.renderer.root).find((item) => item.num === action.target)
+      if (!target || !target.visible || target.isDestroyed)
+        return yield* Effect.fail(new Error(`click target is stale or unavailable: ${action.target}`))
+      if (
+        !Number.isFinite(action.x) ||
+        action.x < 0 ||
+        action.x >= target.width ||
+        !Number.isFinite(action.y) ||
+        action.y < 0 ||
+        action.y >= target.height
+      )
+        return yield* Effect.fail(new Error("click position must be within the target element"))
+      yield* Effect.tryPromise(() => harness.mockMouse.click(target.screenX + action.x, target.screenY + action.y))
       break
+    }
     case "ui.resize":
       if (
         !Number.isSafeInteger(action.cols) ||

packages/simulation/src/frontend/semantics.ts

@@ -0,0 +1,20 @@
+import type { Renderable } from "@opentui/core"
+import type { SimulationProtocol } from "../protocol"
+
+// Semantic renderables set an explicit stable OpenTUI id so ui.state and
+// ui.snapshot expose the same identity. Hierarchy and element handles come
+// from the live render tree.
+export type Definition = Omit<SimulationProtocol.Frontend.SemanticNode, "id" | "element" | "parent">
+
+const key = Symbol.for("opencode.simulation.semantics")
+
+const bind = (definition: () => Definition) => (renderable: Renderable) => {
+  Object.defineProperty(renderable, key, { value: definition, configurable: true })
+}
+
+export const read = (renderable: Renderable) => {
+  const definition: unknown = Reflect.get(renderable, key)
+  return typeof definition === "function" ? (definition as () => Definition) : undefined
+}
+
+export const SimulationSemantics = { bind, read }

packages/simulation/src/frontend/server.ts

@@ -22,6 +22,8 @@ function handle(harness: Harness, request: SimulationProtocol.Frontend.Request)
       return SimulationActions.screenshot(harness, request.params?.name)
     case "ui.state":
       return Effect.sync(() => SimulationActions.state(harness))
+    case "ui.snapshot":
+      return Effect.sync(() => SimulationActions.snapshot(harness))
     case "ui.matches":
       return Effect.sync(() => SimulationActions.matches(harness, request.params.text))
     case "ui.recording.finish":

packages/simulation/src/protocol/index.ts

@@ -67,9 +67,10 @@ export namespace Handshake {
   export const Params = Schema.Struct({
     client: Identity,
     expectedRole: EndpointRole,
-    offeredVersions: Schema.Array(
-      Schema.Int.check(Schema.isGreaterThan(0)),
-    ).check(Schema.isMinLength(1), Schema.isUnique()),
+    offeredVersions: Schema.Array(Schema.Int.check(Schema.isGreaterThan(0))).check(
+      Schema.isMinLength(1),
+      Schema.isUnique(),
+    ),
     requiredCapabilities: Schema.Array(Capability).check(Schema.isUnique()),
     optionalCapabilities: Schema.Array(Capability).check(Schema.isUnique()),
   })
@@ -174,6 +175,7 @@ export namespace Frontend {
     "ui.matches",
     "ui.screenshot",
     "ui.state",
+    "ui.snapshot",
     "ui.capture",
     "ui.recording.finish",
   ] as const satisfies ReadonlyArray<Handshake.Capability>
@@ -221,6 +223,46 @@ export namespace Frontend {
   })
   export interface State extends Schema.Schema.Type<typeof State> {}
 
+  export const SemanticNode = Schema.Struct({
+    id: Schema.NonEmptyString,
+    instance: Schema.optionalKey(Schema.NonEmptyString),
+    parent: Schema.optionalKey(Schema.NonEmptyString),
+    role: Schema.NonEmptyString,
+    label: Schema.optionalKey(Schema.NonEmptyString),
+    element: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)),
+    focused: Schema.optionalKey(Schema.Boolean),
+    selected: Schema.optionalKey(Schema.Boolean),
+    expanded: Schema.optionalKey(Schema.Boolean),
+    disabled: Schema.optionalKey(Schema.Boolean),
+  })
+  export interface SemanticNode extends Schema.Schema.Type<typeof SemanticNode> {}
+
+  export const SemanticSnapshot = Schema.Struct({
+    format: Schema.Literal("opencode-ui-snapshot-v1"),
+    nodes: Schema.Array(SemanticNode).check(
+      Schema.makeFilter((nodes) => {
+        const ids = new Set(nodes.map((node) => node.id))
+        if (ids.size !== nodes.length) return "semantic node ids must be unique"
+        if (new Set(nodes.map((node) => node.element)).size !== nodes.length)
+          return "semantic node elements must be unique"
+        if (nodes.some((node) => node.parent !== undefined && !ids.has(node.parent)))
+          return "semantic node parents must reference another node"
+        const parents = new Map(nodes.map((node) => [node.id, node.parent]))
+        for (const node of nodes) {
+          const visited = new Set<string>()
+          let current: string | undefined = node.id
+          while (current !== undefined) {
+            if (visited.has(current)) return "semantic node hierarchy must be acyclic"
+            visited.add(current)
+            current = parents.get(current)
+          }
+        }
+        return undefined
+      }),
+    ),
+  })
+  export interface SemanticSnapshot extends Schema.Schema.Type<typeof SemanticSnapshot> {}
+
   export const Screenshot = Schema.String
   export type Screenshot = Schema.Schema.Type<typeof Screenshot>
 
@@ -293,7 +335,7 @@ export namespace Frontend {
     }),
     Schema.Struct({
       ...JsonRpc.RequestFields,
-      method: Schema.Literals(["ui.enter", "ui.state", "ui.recording.finish"]),
+      method: Schema.Literals(["ui.enter", "ui.state", "ui.snapshot", "ui.recording.finish"]),
     }),
     Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.capture") }),
   ])