feat(simulation): expose semantic UI snapshots (#37802)

packages/simulation/src/frontend/actions.ts

@@ -9,9 +9,10 @@ import {
   type MockInput,
   type MockMouse,
 } from "@opentui/core/testing"
-import { Config, Effect, FileSystem } from "effect"
-import type { SimulationProtocol } from "../protocol"
+import { Config, Effect, FileSystem, Schema } from "effect"
+import { SimulationProtocol } from "../protocol"
 import { SimulationRenderer } from "./renderer"
+import { SimulationSemantics } from "./semantics"
 
 export type Action = SimulationProtocol.Frontend.Action
 export type Element = SimulationProtocol.Frontend.Element
@@ -60,7 +61,8 @@ function hit(renderer: CliRenderer, renderable: Renderable) {
   if (renderable.width <= 0 || renderable.height <= 0) return false
   const x = Math.floor(renderable.screenX + renderable.width / 2)
   const y = Math.floor(renderable.screenY + renderable.height / 2)
-  return renderer.hitTest(x, y) === renderable.num
+  const target = renderer.hitTest(x, y)
+  return all(renderable).some((item) => item.num === target)
 }
 
 /**
@@ -122,6 +124,25 @@ export function state(harness: Harness) {
   }
 }
 
+export function snapshot(harness: Harness): SimulationProtocol.Frontend.SemanticSnapshot {
+  const ids = new Set<string>()
+  const visit = (renderable: Renderable, parent?: string): SimulationProtocol.Frontend.SemanticNode[] => {
+    if (!renderable.visible || renderable.isDestroyed) return []
+    const definition = SimulationSemantics.read(renderable)?.()
+    if (definition && ids.has(renderable.id)) throw new Error(`duplicate semantic UI id: ${renderable.id}`)
+    if (definition) ids.add(renderable.id)
+    const node = definition
+      ? [{ id: renderable.id, ...definition, ...(parent === undefined ? {} : { parent }), element: renderable.num }]
+      : []
+    const ancestor = definition ? renderable.id : parent
+    return [...node, ...children(renderable).flatMap((child) => visit(child, ancestor))]
+  }
+  return Schema.decodeUnknownSync(SimulationProtocol.Frontend.SemanticSnapshot)({
+    format: "opencode-ui-snapshot-v1",
+    nodes: visit(harness.renderer.root),
+  })
+}
+
 export function matches(harness: Pick<Harness, "screen">, text: string) {
   return harness.screen().includes(text)
 }
@@ -183,9 +204,22 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness
         .find((item) => item.num === action.target)
         ?.focus()
       break
-    case "ui.click":
-      yield* Effect.tryPromise(() => harness.mockMouse.click(action.x, action.y))
+    case "ui.click": {
+      const target = all(harness.renderer.root).find((item) => item.num === action.target)
+      if (!target || !target.visible || target.isDestroyed)
+        return yield* Effect.fail(new Error(`click target is stale or unavailable: ${action.target}`))
+      if (
+        !Number.isFinite(action.x) ||
+        action.x < 0 ||
+        action.x >= target.width ||
+        !Number.isFinite(action.y) ||
+        action.y < 0 ||
+        action.y >= target.height
+      )
+        return yield* Effect.fail(new Error("click position must be within the target element"))
+      yield* Effect.tryPromise(() => harness.mockMouse.click(target.screenX + action.x, target.screenY + action.y))
       break
+    }
     case "ui.resize":
       if (
         !Number.isSafeInteger(action.cols) ||

packages/simulation/src/frontend/semantics.ts

@@ -0,0 +1,20 @@
+import type { Renderable } from "@opentui/core"
+import type { SimulationProtocol } from "../protocol"
+
+// Semantic renderables set an explicit stable OpenTUI id so ui.state and
+// ui.snapshot expose the same identity. Hierarchy and element handles come
+// from the live render tree.
+export type Definition = Omit<SimulationProtocol.Frontend.SemanticNode, "id" | "element" | "parent">
+
+const key = Symbol.for("opencode.simulation.semantics")
+
+const bind = (definition: () => Definition) => (renderable: Renderable) => {
+  Object.defineProperty(renderable, key, { value: definition, configurable: true })
+}
+
+export const read = (renderable: Renderable) => {
+  const definition: unknown = Reflect.get(renderable, key)
+  return typeof definition === "function" ? (definition as () => Definition) : undefined
+}
+
+export const SimulationSemantics = { bind, read }

packages/simulation/src/frontend/server.ts

@@ -22,6 +22,8 @@ function handle(harness: Harness, request: SimulationProtocol.Frontend.Request)
       return SimulationActions.screenshot(harness, request.params?.name)
     case "ui.state":
       return Effect.sync(() => SimulationActions.state(harness))
+    case "ui.snapshot":
+      return Effect.sync(() => SimulationActions.snapshot(harness))
     case "ui.matches":
       return Effect.sync(() => SimulationActions.matches(harness, request.params.text))
     case "ui.recording.finish":