diff --git a/packages/simulation/src/frontend/actions.ts b/packages/simulation/src/frontend/actions.ts index 78f6602ff1..c3958d2e98 100644 --- a/packages/simulation/src/frontend/actions.ts +++ b/packages/simulation/src/frontend/actions.ts @@ -9,9 +9,10 @@ import { type MockInput, type MockMouse, } from "@opentui/core/testing" -import { Config, Effect, FileSystem } from "effect" -import type { SimulationProtocol } from "../protocol" +import { Config, Effect, FileSystem, Schema } from "effect" +import { SimulationProtocol } from "../protocol" import { SimulationRenderer } from "./renderer" +import { SimulationSemantics } from "./semantics" export type Action = SimulationProtocol.Frontend.Action export type Element = SimulationProtocol.Frontend.Element @@ -60,7 +61,8 @@ function hit(renderer: CliRenderer, renderable: Renderable) { if (renderable.width <= 0 || renderable.height <= 0) return false const x = Math.floor(renderable.screenX + renderable.width / 2) const y = Math.floor(renderable.screenY + renderable.height / 2) - return renderer.hitTest(x, y) === renderable.num + const target = renderer.hitTest(x, y) + return all(renderable).some((item) => item.num === target) } /** @@ -122,6 +124,25 @@ export function state(harness: Harness) { } } +export function snapshot(harness: Harness): SimulationProtocol.Frontend.SemanticSnapshot { + const ids = new Set() + const visit = (renderable: Renderable, parent?: string): SimulationProtocol.Frontend.SemanticNode[] => { + if (!renderable.visible || renderable.isDestroyed) return [] + const definition = SimulationSemantics.read(renderable)?.() + if (definition && ids.has(renderable.id)) throw new Error(`duplicate semantic UI id: ${renderable.id}`) + if (definition) ids.add(renderable.id) + const node = definition + ? [{ id: renderable.id, ...definition, ...(parent === undefined ? {} : { parent }), element: renderable.num }] + : [] + const ancestor = definition ? renderable.id : parent + return [...node, ...children(renderable).flatMap((child) => visit(child, ancestor))] + } + return Schema.decodeUnknownSync(SimulationProtocol.Frontend.SemanticSnapshot)({ + format: "opencode-ui-snapshot-v1", + nodes: visit(harness.renderer.root), + }) +} + export function matches(harness: Pick, text: string) { return harness.screen().includes(text) } @@ -183,9 +204,22 @@ export const execute = Effect.fn("SimulationActions.execute")(function* (harness .find((item) => item.num === action.target) ?.focus() break - case "ui.click": - yield* Effect.tryPromise(() => harness.mockMouse.click(action.x, action.y)) + case "ui.click": { + const target = all(harness.renderer.root).find((item) => item.num === action.target) + if (!target || !target.visible || target.isDestroyed) + return yield* Effect.fail(new Error(`click target is stale or unavailable: ${action.target}`)) + if ( + !Number.isFinite(action.x) || + action.x < 0 || + action.x >= target.width || + !Number.isFinite(action.y) || + action.y < 0 || + action.y >= target.height + ) + return yield* Effect.fail(new Error("click position must be within the target element")) + yield* Effect.tryPromise(() => harness.mockMouse.click(target.screenX + action.x, target.screenY + action.y)) break + } case "ui.resize": if ( !Number.isSafeInteger(action.cols) || diff --git a/packages/simulation/src/frontend/semantics.ts b/packages/simulation/src/frontend/semantics.ts new file mode 100644 index 0000000000..91aaa6e88e --- /dev/null +++ b/packages/simulation/src/frontend/semantics.ts @@ -0,0 +1,20 @@ +import type { Renderable } from "@opentui/core" +import type { SimulationProtocol } from "../protocol" + +// Semantic renderables set an explicit stable OpenTUI id so ui.state and +// ui.snapshot expose the same identity. Hierarchy and element handles come +// from the live render tree. +export type Definition = Omit + +const key = Symbol.for("opencode.simulation.semantics") + +const bind = (definition: () => Definition) => (renderable: Renderable) => { + Object.defineProperty(renderable, key, { value: definition, configurable: true }) +} + +export const read = (renderable: Renderable) => { + const definition: unknown = Reflect.get(renderable, key) + return typeof definition === "function" ? (definition as () => Definition) : undefined +} + +export const SimulationSemantics = { bind, read } diff --git a/packages/simulation/src/frontend/server.ts b/packages/simulation/src/frontend/server.ts index f26bb2eaf8..001f7d55df 100644 --- a/packages/simulation/src/frontend/server.ts +++ b/packages/simulation/src/frontend/server.ts @@ -22,6 +22,8 @@ function handle(harness: Harness, request: SimulationProtocol.Frontend.Request) return SimulationActions.screenshot(harness, request.params?.name) case "ui.state": return Effect.sync(() => SimulationActions.state(harness)) + case "ui.snapshot": + return Effect.sync(() => SimulationActions.snapshot(harness)) case "ui.matches": return Effect.sync(() => SimulationActions.matches(harness, request.params.text)) case "ui.recording.finish": diff --git a/packages/simulation/src/protocol/index.ts b/packages/simulation/src/protocol/index.ts index 9a15098333..a99e808b0d 100644 --- a/packages/simulation/src/protocol/index.ts +++ b/packages/simulation/src/protocol/index.ts @@ -67,9 +67,10 @@ export namespace Handshake { export const Params = Schema.Struct({ client: Identity, expectedRole: EndpointRole, - offeredVersions: Schema.Array( - Schema.Int.check(Schema.isGreaterThan(0)), - ).check(Schema.isMinLength(1), Schema.isUnique()), + offeredVersions: Schema.Array(Schema.Int.check(Schema.isGreaterThan(0))).check( + Schema.isMinLength(1), + Schema.isUnique(), + ), requiredCapabilities: Schema.Array(Capability).check(Schema.isUnique()), optionalCapabilities: Schema.Array(Capability).check(Schema.isUnique()), }) @@ -174,6 +175,7 @@ export namespace Frontend { "ui.matches", "ui.screenshot", "ui.state", + "ui.snapshot", "ui.capture", "ui.recording.finish", ] as const satisfies ReadonlyArray @@ -221,6 +223,46 @@ export namespace Frontend { }) export interface State extends Schema.Schema.Type {} + export const SemanticNode = Schema.Struct({ + id: Schema.NonEmptyString, + instance: Schema.optionalKey(Schema.NonEmptyString), + parent: Schema.optionalKey(Schema.NonEmptyString), + role: Schema.NonEmptyString, + label: Schema.optionalKey(Schema.NonEmptyString), + element: Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0)), + focused: Schema.optionalKey(Schema.Boolean), + selected: Schema.optionalKey(Schema.Boolean), + expanded: Schema.optionalKey(Schema.Boolean), + disabled: Schema.optionalKey(Schema.Boolean), + }) + export interface SemanticNode extends Schema.Schema.Type {} + + export const SemanticSnapshot = Schema.Struct({ + format: Schema.Literal("opencode-ui-snapshot-v1"), + nodes: Schema.Array(SemanticNode).check( + Schema.makeFilter((nodes) => { + const ids = new Set(nodes.map((node) => node.id)) + if (ids.size !== nodes.length) return "semantic node ids must be unique" + if (new Set(nodes.map((node) => node.element)).size !== nodes.length) + return "semantic node elements must be unique" + if (nodes.some((node) => node.parent !== undefined && !ids.has(node.parent))) + return "semantic node parents must reference another node" + const parents = new Map(nodes.map((node) => [node.id, node.parent])) + for (const node of nodes) { + const visited = new Set() + let current: string | undefined = node.id + while (current !== undefined) { + if (visited.has(current)) return "semantic node hierarchy must be acyclic" + visited.add(current) + current = parents.get(current) + } + } + return undefined + }), + ), + }) + export interface SemanticSnapshot extends Schema.Schema.Type {} + export const Screenshot = Schema.String export type Screenshot = Schema.Schema.Type @@ -293,7 +335,7 @@ export namespace Frontend { }), Schema.Struct({ ...JsonRpc.RequestFields, - method: Schema.Literals(["ui.enter", "ui.state", "ui.recording.finish"]), + method: Schema.Literals(["ui.enter", "ui.state", "ui.snapshot", "ui.recording.finish"]), }), Schema.Struct({ ...JsonRpc.RequestFields, method: Schema.Literal("ui.capture") }), ]) diff --git a/packages/simulation/test/actions.test.ts b/packages/simulation/test/actions.test.ts index 98976d0e74..9849a70628 100644 --- a/packages/simulation/test/actions.test.ts +++ b/packages/simulation/test/actions.test.ts @@ -1,6 +1,9 @@ import { expect, test } from "bun:test" +import { BoxRenderable, TextRenderable } from "@opentui/core" import { Effect } from "effect" -import { execute, type Harness, matches } from "../src/frontend/actions" +import { createHarness, execute, type Harness, matches, snapshot, state } from "../src/frontend/actions" +import { SimulationRenderer } from "../src/frontend/renderer" +import { SimulationSemantics } from "../src/frontend/semantics" test("matches literal screen text", () => { const harness = { screen: () => "OpenCode [ready].*" } @@ -39,3 +42,144 @@ test("normalizes named keys for OpenTUI", async () => { ["x", undefined], ]) }) + +test("clicks a target at relative coordinates through descendant text", async () => { + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const renderer = yield* SimulationRenderer.create({}) + let clicks = 0 + const button = new BoxRenderable(renderer, { + id: "permission.action.once", + width: 12, + height: 1, + onMouseUp: () => clicks++, + }) + button.add(new TextRenderable(renderer, { content: "Allow once" })) + renderer.root.add(button) + const harness = createHarness(renderer) + yield* Effect.promise(() => harness.renderOnce()) + + expect(state(harness).elements).toContainEqual(expect.objectContaining({ id: button.id, clickable: true })) + yield* execute(harness, { type: "ui.click", target: button.num, x: 1, y: 0 }) + expect(clicks).toBe(1) + + renderer.root.remove(button) + const error = yield* execute(harness, { type: "ui.click", target: button.num, x: 1, y: 0 }).pipe(Effect.flip) + expect(error.message).toContain("click target is stale or unavailable") + }), + ), + ) +}) + +test("snapshots lazy semantic hierarchy and interaction state", async () => { + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const renderer = yield* SimulationRenderer.create({}) + let selected = "once" + const dialog = new BoxRenderable(renderer, { id: "session.permission" }) + const actions = new BoxRenderable(renderer, { id: "session.permission.actions" }) + const once = new BoxRenderable(renderer, { id: "session.permission.action.once" }) + SimulationSemantics.bind(() => ({ + instance: "permission-1", + role: "dialog", + label: "Permission required", + expanded: false, + }))(dialog) + SimulationSemantics.bind(() => ({ + instance: "permission-1", + role: "listbox", + label: "Permission choices", + }))(actions) + SimulationSemantics.bind(() => ({ + instance: "permission-1", + role: "option", + label: "Allow once", + focused: selected === "once", + selected: selected === "once", + disabled: false, + }))(once) + renderer.root.add(dialog) + dialog.add(actions) + actions.add(once) + + expect(snapshot(createHarness(renderer))).toEqual({ + format: "opencode-ui-snapshot-v1", + nodes: [ + { + id: "session.permission", + instance: "permission-1", + role: "dialog", + label: "Permission required", + element: dialog.num, + expanded: false, + }, + { + id: "session.permission.actions", + instance: "permission-1", + parent: "session.permission", + role: "listbox", + label: "Permission choices", + element: actions.num, + }, + { + id: "session.permission.action.once", + instance: "permission-1", + parent: "session.permission.actions", + role: "option", + label: "Allow once", + element: once.num, + focused: true, + selected: true, + disabled: false, + }, + ], + }) + + selected = "reject" + expect(snapshot(createHarness(renderer)).nodes.at(-1)).toMatchObject({ + id: "session.permission.action.once", + focused: false, + selected: false, + }) + dialog.visible = false + expect(snapshot(createHarness(renderer)).nodes).toEqual([]) + }), + ), + ) +}) + +test("rejects duplicate semantic identities", async () => { + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const renderer = yield* SimulationRenderer.create({}) + const first = new BoxRenderable(renderer, { id: "duplicate" }) + const second = new BoxRenderable(renderer, { id: "duplicate" }) + const definition = () => ({ role: "option" }) + SimulationSemantics.bind(definition)(first) + SimulationSemantics.bind(definition)(second) + renderer.root.add(first) + renderer.root.add(second) + + expect(() => snapshot(createHarness(renderer))).toThrow("duplicate semantic UI id: duplicate") + }), + ), + ) +}) + +test("validates lazy semantic definitions before returning them", async () => { + await Effect.runPromise( + Effect.scoped( + Effect.gen(function* () { + const renderer = yield* SimulationRenderer.create({}) + const invalid = new BoxRenderable(renderer, { id: "invalid" }) + SimulationSemantics.bind(() => ({ role: "" }))(invalid) + renderer.root.add(invalid) + + expect(() => snapshot(createHarness(renderer))).toThrow() + }), + ), + ) +}) diff --git a/packages/simulation/test/frontend-server.test.ts b/packages/simulation/test/frontend-server.test.ts index 849adfb6f7..209596c848 100644 --- a/packages/simulation/test/frontend-server.test.ts +++ b/packages/simulation/test/frontend-server.test.ts @@ -39,7 +39,7 @@ test("scopes the frontend control server and reports malformed JSON", async () = protocolVersion: 1, role: "ui", server: { name: "opencode", version: expect.any(String) }, - capabilities: expect.arrayContaining(["ui.state", "ui.capture"]), + capabilities: expect.arrayContaining(["ui.state", "ui.snapshot", "ui.capture"]), }, }) @@ -60,6 +60,13 @@ test("scopes the frontend control server and reports malformed JSON", async () = }, }) + socket.send(JSON.stringify({ jsonrpc: "2.0", id: 3, method: "ui.snapshot" })) + expect(yield* Queue.take(messages)).toEqual({ + jsonrpc: "2.0", + id: 3, + result: { format: "opencode-ui-snapshot-v1", nodes: [] }, + }) + socket.send("{") expect(yield* Queue.take(messages)).toMatchObject({ id: null, diff --git a/packages/simulation/test/protocol.test.ts b/packages/simulation/test/protocol.test.ts index bd516bea99..93c4e560ea 100644 --- a/packages/simulation/test/protocol.test.ts +++ b/packages/simulation/test/protocol.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test" -import { Effect } from "effect" +import { Effect, Schema } from "effect" import { Backend, Frontend, Handshake } from "../src/protocol" test("decodes ui.matches text params", () => { @@ -21,6 +21,53 @@ test("decodes ui.matches text params", () => { ).toThrow() }) +test("decodes semantic UI snapshots", () => { + expect( + Frontend.decodeRequest({ + jsonrpc: "2.0", + id: 1, + method: "ui.snapshot", + }), + ).toMatchObject({ method: "ui.snapshot" }) + const decode = Schema.decodeUnknownSync(Frontend.SemanticSnapshot) + expect( + decode({ + format: "opencode-ui-snapshot-v1", + nodes: [ + { + id: "session.permission", + role: "dialog", + label: "Permission required", + element: 1, + expanded: false, + }, + ], + }), + ).toMatchObject({ nodes: [{ role: "dialog", expanded: false }] }) + expect(() => + decode({ + format: "opencode-ui-snapshot-v1", + nodes: [{ id: "", role: "dialog", element: 0 }], + }), + ).toThrow() + for (const nodes of [ + [ + { id: "duplicate", role: "dialog", element: 1 }, + { id: "duplicate", role: "option", element: 2 }, + ], + [ + { id: "first", role: "dialog", element: 1 }, + { id: "second", role: "option", element: 1 }, + ], + [{ id: "orphan", parent: "missing", role: "option", element: 1 }], + [ + { id: "first", parent: "second", role: "dialog", element: 1 }, + { id: "second", parent: "first", role: "option", element: 2 }, + ], + ]) + expect(() => decode({ format: "opencode-ui-snapshot-v1", nodes })).toThrow() +}) + const params: Handshake.Params = { client: { name: "opencode-drive", version: "test" }, expectedRole: "ui", diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index a6cf7532fc..5b8b09ee9c 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -957,7 +957,14 @@ export function Session() { {null} 0}> - + + {(_) => { + const request = permissions()[0] + return request ? ( + + ) : null + }} + 0}> @@ -1460,7 +1467,8 @@ function CompactionMessage(props: { message: Extract props.message.status const cancelled = () => props.message.status === "failed" && props.message.error.type === "aborted" - const text = () => (props.message.status === "failed" ? (cancelled() ? "" : props.message.error.message) : props.message.summary) + const text = () => + props.message.status === "failed" ? (cancelled() ? "" : props.message.error.message) : props.message.summary const content = createMemo(() => text().trim()) const color = () => (status() === "failed" && !cancelled() ? themeV2.text.feedback.error() : themeV2.text.subdued()) return ( @@ -1807,7 +1815,9 @@ function AssistantMessage(props: { message: SessionMessageAssistant; last: boole - + {Locale.titlecase(props.message.agent)} ยท {model()} @@ -2360,9 +2370,7 @@ function BlockTool(props: { paddingBottom={1} paddingLeft={2} gap={1} - backgroundColor={ - hover() ? themeV2.raise(themeV2.background()) : themeV2.background() - } + backgroundColor={hover() ? themeV2.raise(themeV2.background()) : themeV2.background()} customBorderChars={SplitBorder.customBorderChars} borderColor={themeV2.background()} onMouseOver={() => props.onClick && setHover(true)} @@ -2379,9 +2387,13 @@ function BlockTool(props: { {(title) => ( {title()}} + fallback={ + {title()} + } > - {title().replace(/^# /, "")} + + {title().replace(/^# /, "")} + )} diff --git a/packages/tui/src/routes/session/permission.tsx b/packages/tui/src/routes/session/permission.tsx index 3b3ecf41af..0ce5f04dc0 100644 --- a/packages/tui/src/routes/session/permission.tsx +++ b/packages/tui/src/routes/session/permission.tsx @@ -15,6 +15,7 @@ import { getScrollAcceleration } from "../../util/scroll" import { useConfig } from "../../config" import { Keymap } from "../../context/keymap" import { usePathFormatter } from "../../context/path-format" +import { SimulationSemantics } from "../../simulation/semantics" type PermissionStage = "permission" | "always" | "reject" @@ -160,6 +161,8 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director @@ -167,7 +170,9 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director - This will allow the following patterns until OpenCode is restarted + + This will allow the following patterns until OpenCode is restarted + {(pattern) => ( @@ -197,6 +202,8 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director { void client.api.permission.reply({ sessionID: props.request.sessionID, @@ -425,6 +432,8 @@ export function PermissionPrompt(props: { request: PermissionV2Request; director const body = ( void; onCancel: () => void }) { +export function permissionSemanticLabel(action: string, title?: string) { + return `Permission required: ${title ?? action}` +} + +function RejectPrompt(props: { + action: string + instance: string + onConfirm: (message: string) => void + onCancel: () => void +}) { let input: TextareaRenderable const { themeV2 } = useTheme().contextual("elevated") const dimensions = useTerminalDimensions() @@ -495,6 +513,12 @@ function RejectPrompt(props: { onConfirm: (message: string) => void; onCancel: ( return ( ({ + instance: props.instance, + role: "dialog", + label: `Reject permission: ${props.action}`, + }))} backgroundColor={themeV2.background()} border={["left"]} borderColor={themeV2.text.feedback.error()} @@ -522,8 +546,16 @@ function RejectPrompt(props: { onConfirm: (message: string) => void; onCancel: ( gap={1} >