From 4205af5cded82985e9f8f09b2fd558356b8668ff Mon Sep 17 00:00:00 2001 From: LukeParkerDev <10430890+Hona@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:41:13 +1000 Subject: [PATCH] feat(client): add shared Chromium driver --- packages/client/README.md | 54 +- packages/client/src/node/browser/chromium.ts | 696 +++++++++++++++++++ packages/client/src/node/browser/driver.ts | 6 + packages/client/src/node/index.ts | 1 + packages/client/test/node/chromium.test.ts | 85 +++ packages/client/test/node/package-smoke.ts | 133 ++++ packages/client/test/types/node-consumer.ts | 14 +- 7 files changed, 985 insertions(+), 4 deletions(-) create mode 100644 packages/client/src/node/browser/chromium.ts create mode 100644 packages/client/test/node/chromium.test.ts create mode 100644 packages/client/test/node/package-smoke.ts diff --git a/packages/client/README.md b/packages/client/README.md index 8c53e47c7e..7361dec894 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -1,17 +1,67 @@ # @opencode-ai/client -Private generation target for clients derived directly from OpenCode's authoritative Effect `HttpApi`. +Promise and Effect clients derived from OpenCode's authoritative Effect `HttpApi`, plus handwritten Node transports. ## Entrypoints - `@opencode-ai/client`: zero-Effect Promise client using `fetch`. +- `@opencode-ai/client/node`: Promise client plus Node-hosted browser attachments. - `@opencode-ai/client/effect`: rich Effect network client using an environment-provided `HttpClient`. The generated surface includes every standard HTTP group from Server's concrete API. The build compiler reads `@opencode-ai/server/api`; the generated Effect runtime imports a client-local projection built from Protocol, with a generation-equivalence test preventing transport drift. Custom transports such as the PTY WebSocket connection remain outside the generic HTTP client. Run `bun run generate` after changing the contract and `bun run check:generated` to detect committed-output drift. The Effect entrypoint uses canonical decoded values such as `Session.ID`, `Location.Ref`, and `Prompt`. These datatypes come from the lightweight `@opencode-ai/schema` package and are re-exported so callers depend only on the client surface. Protocol owns endpoint construction and middleware placement; Server supplies the concrete middleware keys used by the build-time API. -The Promise root remains structural and has no Core or Effect runtime dependency. `/effect` depends only on Effect, Schema, and Protocol and is browser-bundle safe. Bundle-boundary tests enforce both import graphs. +The Promise root remains structural and has no Core, Effect, Schema, Protocol, or WebSocket runtime dependency. `/node` adds Effect, Schema, Protocol, and `ws`, but never Core or Server. `/effect` depends only on Effect, Schema, and Protocol and is browser-bundle safe. Bundle-boundary tests enforce these import graphs. + +## Node browser attachments + +The Node entrypoint owns the control connection, Session lease, authenticated proxy, and network tunnels. Chromium hosts provide a small platform port once with `BrowserDriver.chromium`; the SDK owns command semantics, CDP input, snapshots, generations, cancellation, and limits. + +```ts +import { BrowserDriver, OpenCode } from "@opencode-ai/client/node" + +const driver = BrowserDriver.chromium(async ({ proxy, signal }) => { + const view = await createChromiumView({ proxy, signal }) + return { + resource: view, + state: () => view.state(), + subscribe: (listener) => view.subscribe((state, mainDocumentChanged) => listener({ state, mainDocumentChanged })), + navigate: (url) => view.navigate(url), + back: () => view.back(), + forward: () => view.forward(), + reload: () => view.reload(), + stop: () => view.stop(), + send: (method, params) => view.sendCDP(method, params), + viewport: () => view.viewport(), + screenshot: ({ maxDimension }) => view.capturePNG({ maxDimension }), + dispose: () => view.close(), + } +}) + +const client = OpenCode.make({ + baseUrl: "https://opencode.example", + headers: { authorization: `Basic ${credentials}` }, +}) +const registration = await client.browser.register({ + sessionID, + open: () => showBrowserPane(), +}) +const attachment = await registration.attach({ driver }) + +await attachment.resource.navigate("example.com") +const view = attachment.resource.resource +await attachment.close() +await registration.close() +``` + +`register` owns one control WebSocket for one Session. Its `open` callback is invoked when the server requests the browser pane. `attach` resolves only after the server acknowledges the exact lease; closing an attachment leaves its registration connected for a later `open`, while closing the registration closes the socket. + +Driver factories should return after configuring their resource rather than await a proxied navigation: tunnel dialing is deliberately held behind the first lease acknowledgement, which is published after the driver supplies its initial state. + +Port state events set `mainDocumentChanged` only when the main document changes; this advances the public generation and invalidates element refs. `send` must dispatch CDP calls in invocation order. `screenshot` returns PNG bytes and dimensions, proportionally scaled to `maxDimension` without upscaling. The returned controller serializes local navigation with remote commands; `stop` immediately interrupts active work, and controller disposal is idempotent. An aborted or timed-out operation that reached the platform disposes the port so late native completion cannot cross the queue fence. + +`BrowserDriver` descriptors are structural factory functions, so adapters remain compatible across duplicate client package instances. The Node entrypoint also re-exports canonical `Browser` contracts. `BrowserDriver.define` remains the advanced escape hatch for non-Chromium semantics; throw `BrowserDriverError` for typed command failures there. Structurally equivalent errors are accepted only when their `code` is a valid `Browser.ErrorCode`. Effect consumers construct canonical decoded inputs: diff --git a/packages/client/src/node/browser/chromium.ts b/packages/client/src/node/browser/chromium.ts new file mode 100644 index 0000000000..f0a4a05749 --- /dev/null +++ b/packages/client/src/node/browser/chromium.ts @@ -0,0 +1,696 @@ +import { Browser } from "@opencode-ai/schema/browser" +import { + BrowserDriverError, + type BrowserDriver, + type BrowserDriverContext, + type BrowserDriverInstance, +} from "./driver.js" + +type ChromiumViewState = Omit +type ChromiumViewEvent = { readonly state: ChromiumViewState; readonly mainDocumentChanged: boolean } +type ChromiumCommands = { + "Runtime.evaluate": { readonly expression: string } + "Runtime.callFunctionOn": { + readonly objectId: string + readonly functionDeclaration: string + readonly arguments?: ReadonlyArray<{ readonly value: string }> + readonly returnByValue: true + } + "Runtime.releaseObject": { readonly objectId: string } + "Input.dispatchMouseEvent": { + readonly type: "mouseMoved" | "mousePressed" | "mouseReleased" | "mouseWheel" + readonly x: number + readonly y: number + readonly button?: "left" + readonly clickCount?: 1 + readonly deltaX?: number + readonly deltaY?: number + } + "Input.dispatchKeyEvent": { + readonly type: "keyDown" | "keyUp" + readonly key: string + readonly code: string + readonly modifiers?: number + readonly windowsVirtualKeyCode?: number + } + "Input.insertText": { readonly text: string } +} +type ChromiumCommand = { + [Method in keyof ChromiumCommands]: { readonly method: Method; readonly params: ChromiumCommands[Method] } +}[keyof ChromiumCommands] + +/** Platform primitives used by the shared semantic driver. */ +export interface ChromiumPort { + readonly resource: Resource; readonly state: () => ChromiumViewState + readonly subscribe: (listener: (event: ChromiumViewEvent) => void) => () => void + readonly navigate: (url: string) => PromiseLike; readonly back: () => PromiseLike | void + readonly forward: () => PromiseLike | void; readonly reload: () => PromiseLike | void + readonly stop: () => void; readonly send: (command: ChromiumCommand) => PromiseLike + readonly viewport: () => { readonly width: number; readonly height: number } + readonly screenshot: (maxDimension: number) => PromiseLike<{ + readonly data: Uint8Array + readonly width: number + readonly height: number + }> + readonly dispose: () => PromiseLike | void +} + +export interface ChromiumController extends AsyncDisposable { + readonly resource: Resource; readonly state: () => Browser.State + readonly subscribe: (listener: (state: Browser.State) => void) => () => void + readonly navigate: (url: string) => Promise; readonly back: () => Promise + readonly forward: () => Promise; readonly reload: () => Promise + readonly stop: () => void; readonly dispose: () => Promise +} + +export type ChromiumDriver = BrowserDriver> + +type SnapshotNode = { + readonly token?: string; readonly role: string; readonly name: string; readonly value: string; readonly depth: number + readonly checked?: boolean; readonly disabled?: boolean; readonly expanded?: boolean; readonly selected?: boolean +} + +type Page = { + readonly port: ChromiumPort; readonly lifetime: AbortSignal + readonly refs: Set; readonly listeners: Set<(state: Browser.State) => void> + state: ChromiumViewState; document: number; nextRef: number; snapshotObjectID?: string + active?: AbortController; unsubscribe?: () => void; queue: Promise + disposed: boolean; disposal?: Promise +} + +const commandTimeout = 10_000 +const snapshotLimit = 500 +const screenshotDimensionLimit = 2_000 +const screenshotByteLimit = 5 * 1_024 * 1_024 +export function chromiumDriver( + create: (context: BrowserDriverContext) => PromiseLike> | ChromiumPort, +): ChromiumDriver { + return async (context) => { + const port = await create(context) + if (!context.signal.aborted) return createInstance(port, context.signal) + await Promise.resolve(port.dispose()) + throw context.signal.reason instanceof Error ? context.signal.reason : new Error("Chromium driver creation was aborted") + } +} +function createInstance( + port: ChromiumPort, + lifetime: AbortSignal, +): BrowserDriverInstance> { + const page: Page = { + port, + lifetime, + refs: new Set(), + listeners: new Set(), + state: port.state(), + document: 0, + nextRef: 0, + queue: Promise.resolve(), + disposed: false, + } + page.unsubscribe = port.subscribe((event) => { + if (page.disposed) return + if (event.mainDocumentChanged) { + page.document++ + invalidateRefs(page) + } + page.state = event.state + publish(page) + }) + + const dispose = () => disposePage(page) + const controller: ChromiumController = Object.freeze({ + resource: port.resource, + state: () => state(page), + subscribe: (listener) => subscribe(page, listener), + navigate: (url) => schedule(page, undefined, (signal) => navigate(page, url, signal)), + back: () => localAction(page, () => port.back()), + forward: () => localAction(page, () => port.forward()), + reload: () => localAction(page, () => port.reload()), + stop: () => stop(page), + dispose, + [Symbol.asyncDispose]: dispose, + }) + return Object.freeze({ + resource: controller, + state: controller.state, + subscribe: controller.subscribe, + execute: (command: Browser.Command, options: { readonly signal: AbortSignal }) => + schedule(page, options.signal, (signal) => execute(page, command, signal)), + dispose, + }) +} +async function execute(page: Page, command: Browser.Command, signal: AbortSignal) { + assertDocument(page, command.generation) + switch (command.type) { + case "navigate": + await navigate(page, command.url, signal) + return { type: "navigate", state: state(page) } as const + case "snapshot": + return snapshot(page, command.generation, signal) + case "click": + await click(page, command.ref, command.generation, signal) + return { type: "click", state: refresh(page) } as const + case "fill": + await fill(page, command.ref, command.text, command.generation, signal) + return { type: "fill", state: refresh(page) } as const + case "press": + await press(page, command.key, command.generation, signal) + return { type: "press", state: refresh(page) } as const + case "scroll": + await scroll(page, command.direction, command.pixels, command.generation, signal) + return { type: "scroll", state: refresh(page) } as const + case "screenshot": + return screenshot(page, command.generation, signal) + } +} +async function navigate(page: Page, input: string, signal: AbortSignal) { + const url = normalizeURL(input) + const onAbort = () => page.port.stop() + signal.addEventListener("abort", onAbort, { once: true }) + await bounded(() => page.port.navigate(url), signal, 30_000, "The browser navigation timed out.") + .catch((error) => { + if (signal.aborted) throw error + if (error instanceof BrowserDriverError) throw error + throw browserError("navigation_failed", error instanceof Error ? error.message : String(error)) + }) + .finally(() => signal.removeEventListener("abort", onAbort)) + refresh(page) +} +async function snapshot(page: Page, generation: number, signal: AbortSignal) { + const object = await send( + page, + { method: "Runtime.evaluate", params: { expression: snapshotExpression(page.nextRef) } }, + signal, + ) + const objectID = runtimeObjectID(object) + const result = await send( + page, + { + method: "Runtime.callFunctionOn", + params: { + objectId: objectID, + functionDeclaration: "function() { return this.result }", + returnByValue: true, + }, + }, + signal, + ) + .then(readSnapshot) + .catch((error) => { + release(page, objectID) + throw error + }) + assertDocument(page, generation) + invalidateRefs(page) + page.snapshotObjectID = objectID + page.nextRef = Math.max(page.nextRef, result.nextRef) + const lines = result.nodes.map((node) => { + if (node.token) page.refs.add(node.token) + const flags = [ + node.checked === undefined ? undefined : `checked=${node.checked}`, + node.disabled === undefined ? undefined : `disabled=${node.disabled}`, + node.expanded === undefined ? undefined : `expanded=${node.expanded}`, + node.selected === undefined ? undefined : `selected=${node.selected}`, + ].filter((value): value is string => value !== undefined) + const details = [ + node.name ? JSON.stringify(node.name) : undefined, + node.value && node.value !== node.name ? `value=${JSON.stringify(node.value)}` : undefined, + ].filter((value): value is string => value !== undefined) + return `${" ".repeat(node.depth)}${node.token ? `${node.token} ` : ""}[${node.role}]${details.length ? ` ${details.join(" ")}` : ""}${flags.length ? ` ${flags.join(" ")}` : ""}` + }) + const current = page.port.state() + const content = [ + `Page: ${current.title.replaceAll(/\s+/g, " ").trim().slice(0, 1_024)}`, + `URL: ${current.url.slice(0, 16_384)}`, + "", + ...lines, + ] + .join("\n") + .slice(0, 40 * 1_024) + return { type: "snapshot", state: refresh(page), format: "opencode.semantic.v1", content } as const +} +async function click(page: Page, ref: Browser.Ref, generation: number, signal: AbortSignal) { + const objectID = resolveRef(page, ref) + const point = await send( + page, + { + method: "Runtime.callFunctionOn", + params: { + objectId: objectID, + functionDeclaration: + "function(token) { const element = this.refs[token]; if (!element || !element.isConnected) throw new Error('stale element'); element.scrollIntoView({ block: 'center', inline: 'center' }); const bounds = element.getBoundingClientRect(); if (bounds.width <= 0 || bounds.height <= 0) throw new Error('element has no bounds'); return { x: bounds.left + bounds.width / 2, y: bounds.top + bounds.height / 2 } }", + arguments: [{ value: ref }], + returnByValue: true, + }, + }, + signal, + ).then(readPoint) + assertDocument(page, generation) + await send(page, { method: "Input.dispatchMouseEvent", params: { type: "mouseMoved", ...point } }, signal) + await inputPair( + () => + send( + page, + { + method: "Input.dispatchMouseEvent", + params: { type: "mousePressed", button: "left", clickCount: 1, ...point }, + }, + signal, + ), + () => + send(page, { + method: "Input.dispatchMouseEvent", + params: { type: "mouseReleased", button: "left", clickCount: 1, ...point }, + }), + ) + assertDocument(page, generation) +} +async function fill( + page: Page, + ref: Browser.Ref, + text: string, + generation: number, + signal: AbortSignal, +) { + const editable = await send( + page, + { + method: "Runtime.callFunctionOn", + params: { + objectId: resolveRef(page, ref), + functionDeclaration: fillFunction, + arguments: [{ value: ref }], + returnByValue: true, + }, + }, + signal, + ).then(runtimeValue) + assertDocument(page, generation) + if (editable !== true) throw browserError("stale_ref", "The browser element is not editable. Call browser_snapshot again.") + const select = { key: "a", code: "KeyA", modifiers: process.platform === "darwin" ? 4 : 2 } + await keyPair(page, select, signal) + await keyPair(page, { key: "Backspace", code: "Backspace", windowsVirtualKeyCode: 8 }, signal) + await send(page, { method: "Input.insertText", params: { text } }, signal) + assertDocument(page, generation) +} +async function press(page: Page, key: Browser.Key, generation: number, signal: AbortSignal) { + await keyPair(page, keyInfo(key), signal) + assertDocument(page, generation) +} +async function scroll( + page: Page, + direction: Browser.Direction, + pixels: number, + generation: number, + signal: AbortSignal, +) { + const viewport = page.port.viewport() + const distance = Math.min(2_000, Math.max(1, pixels)) + await send( + page, + { + method: "Input.dispatchMouseEvent", + params: { + type: "mouseWheel", + x: Math.max(0, Math.round(viewport.width / 2)), + y: Math.max(0, Math.round(viewport.height / 2)), + deltaX: direction === "left" ? -distance : direction === "right" ? distance : 0, + deltaY: direction === "up" ? -distance : direction === "down" ? distance : 0, + }, + }, + signal, + ) + assertDocument(page, generation) +} +async function screenshot(page: Page, generation: number, signal: AbortSignal) { + const source = await bounded( + () => page.port.screenshot(screenshotDimensionLimit), + signal, + commandTimeout, + "The browser screenshot timed out.", + ) + assertDocument(page, generation) + if (source.data.byteLength > screenshotByteLimit) { + throw browserError("result_too_large", "The browser screenshot exceeds 5 MiB.") + } + if ( + !Number.isSafeInteger(source.width) || + !Number.isSafeInteger(source.height) || + source.width < 1 || + source.height < 1 || + source.width > screenshotDimensionLimit || + source.height > screenshotDimensionLimit + ) { + throw browserError("internal", "The browser pane has no drawable area.") + } + return { + type: "screenshot", + state: refresh(page), + mediaType: "image/png", + data: new Uint8Array(source.data), + width: source.width, + height: source.height, + } as const +} +function schedule( + page: Page, + signal: AbortSignal | undefined, + run: (signal: AbortSignal) => Promise, +) { + assertAttached(page) + throwIfAborted(signal) + const result = page.queue.then(async () => { + assertAttached(page) + throwIfAborted(signal) + const controller = new AbortController() + page.active = controller + const combined = AbortSignal.any([page.lifetime, controller.signal, ...(signal ? [signal] : [])]) + return run(combined).finally(() => { + if (page.active === controller) page.active = undefined + }) + }) + page.queue = result.then( + () => undefined, + () => undefined, + ) + return result.catch((error) => { + throw normalizeError(error) + }) +} +function localAction(page: Page, run: () => PromiseLike | void) { + return schedule(page, undefined, async (signal) => { + throwIfAborted(signal) + await run() + throwIfAborted(signal) + }) +} +function stop(page: Page) { + assertAttached(page) + page.active?.abort() + try { + page.port.stop() + } catch (error) { + throw normalizeError(error) + } +} +function subscribe(page: Page, listener: (state: Browser.State) => void) { + assertAttached(page) + page.listeners.add(listener) + listener(state(page)) + return () => page.listeners.delete(listener) +} +function publish(page: Page) { + const current = state(page) + page.listeners.forEach((listener) => listener(current)) +} +function refresh(page: Page) { + page.state = page.port.state() + publish(page) + return state(page) +} +function state(page: Page): Browser.State { + assertAttached(page) + return { + url: page.state.url.slice(0, 16_384), + title: page.state.title.slice(0, 1_024), + loading: page.state.loading, + canGoBack: page.state.canGoBack, + canGoForward: page.state.canGoForward, + generation: page.document, + } +} +function disposePage(page: Page) { + if (page.disposal) return page.disposal + page.disposed = true + page.active?.abort() + page.listeners.clear() + invalidateRefs(page) + page.unsubscribe?.() + page.unsubscribe = undefined + page.port.stop() + page.disposal = Promise.resolve(page.port.dispose()) + return page.disposal +} +function invalidateRefs(page: Page) { + if (page.snapshotObjectID) release(page, page.snapshotObjectID) + page.snapshotObjectID = undefined + page.refs.clear() +} +function release(page: Page, objectID: string) { + void Promise.resolve( + page.port.send({ method: "Runtime.releaseObject", params: { objectId: objectID } }), + ).catch(() => undefined) +} +function resolveRef(page: Page, ref: Browser.Ref) { + if (!page.snapshotObjectID || !page.refs.has(ref)) { + throw browserError("stale_ref", "The element reference is stale. Call browser_snapshot again.") + } + return page.snapshotObjectID +} +function send(page: Page, command: ChromiumCommand, signal?: AbortSignal) { + return bounded( + () => page.port.send(command), + signal, + commandTimeout, + "The browser command timed out.", + ).catch((error) => { + if (staleProtocolError(error)) { + throw browserError("stale_ref", "The element reference is stale. Call browser_snapshot again.") + } + throw error + }) +} +function readSnapshot(input: unknown) { + const value = runtimeValue(input) + if (!record(value) || !Array.isArray(value.nodes) || value.nodes.length > snapshotLimit) { + throw browserError("internal", "Invalid browser snapshot response.") + } + if (!Number.isSafeInteger(value.nextRef) || Number(value.nextRef) < 0) { + throw browserError("internal", "Invalid browser snapshot response.") + } + const nodes = value.nodes.map((node) => { + if (!snapshotNode(node)) throw browserError("internal", "Invalid browser snapshot response.") + return node + }) + return { nodes, nextRef: Number(value.nextRef) } +} +function snapshotNode(input: unknown): input is SnapshotNode { + if (!record(input)) return false + if ( + typeof input.role !== "string" || + !/^[a-zA-Z0-9_-]{1,40}$/.test(input.role) || + typeof input.name !== "string" || + typeof input.value !== "string" || + !Number.isSafeInteger(input.depth) || + Number(input.depth) < 0 || + Number(input.depth) > 6 + ) + return false + if (input.token !== undefined && (typeof input.token !== "string" || !/^e[1-9][0-9]*$/.test(input.token))) return false + return true +} +function runtimeObjectID(input: unknown) { + if (!record(input) || !record(input.result) || typeof input.result.objectId !== "string") { + throw browserError("internal", "Browser page operation failed.") + } + return input.result.objectId +} +function runtimeValue(input: unknown) { + if (!record(input)) throw browserError("internal", "Browser page operation failed.") + if (input.exceptionDetails !== undefined) { + const details = record(input.exceptionDetails) ? input.exceptionDetails : undefined + const exception = details && record(details.exception) ? details.exception : undefined + const message = + (exception && typeof exception.description === "string" && exception.description) || + (details && typeof details.text === "string" && details.text) || + "Browser page operation failed." + if (staleProtocolError(message)) { + throw browserError("stale_ref", "The element reference is stale. Call browser_snapshot again.") + } + throw browserError("internal", message) + } + if (!record(input.result) || !("value" in input.result)) { + throw browserError("internal", "Browser page operation failed.") + } + return input.result.value +} +function readPoint(input: unknown) { + const value = runtimeValue(input) + if (!record(value) || typeof value.x !== "number" || typeof value.y !== "number") { + throw browserError("stale_ref", "The browser element has no clickable bounds.") + } + return { x: value.x, y: value.y } +} +function keyPair( + page: Page, + key: { readonly key: string; readonly code: string; readonly modifiers?: number; readonly windowsVirtualKeyCode?: number }, + signal: AbortSignal, +) { + return inputPair( + () => send(page, { method: "Input.dispatchKeyEvent", params: { type: "keyDown", ...key } }, signal), + () => send(page, { method: "Input.dispatchKeyEvent", params: { type: "keyUp", ...key } }), + ) +} +async function inputPair(down: () => Promise, up: () => Promise) { + try { + await down() + } finally { + await up() + } +} +function keyInfo(key: Browser.Key) { + const codes: Partial> = { + Enter: 13, + Tab: 9, + Escape: 27, + Backspace: 8, + Delete: 46, + Space: 32, + } + const windowsVirtualKeyCode = codes[key] + return { + key: key === "Space" ? " " : key, + code: key, + ...(windowsVirtualKeyCode ? { windowsVirtualKeyCode } : {}), + } +} +function assertDocument(page: Page, generation: number) { + if (page.document !== generation) { + throw browserError("stale_ref", "The browser page changed. Call browser_snapshot again.") + } +} +function assertAttached(page: Page) { + if (page.disposed) throw browserError("not_attached", "The browser page is no longer attached.") +} +function throwIfAborted(signal?: AbortSignal) { + if (signal?.aborted) throw browserError("aborted", "The browser action was aborted.") +} +function browserError(code: Browser.ErrorCode, message: string) { + return new BrowserDriverError(code, message.slice(0, 1_024)) +} +function normalizeError(error: unknown) { + if (error instanceof BrowserDriverError) return error + return browserError("internal", error instanceof Error ? error.message : String(error)) +} +function normalizeURL(input: string) { + const value = input.trim() + if (value.length > 16_384) throw browserError("invalid_url", "The browser URL is too long.") + if (!value || value === "about:blank") return "about:blank" + const candidate = /^(localhost|127(?:\.\d{1,3}){3}|\[?::1\]?)(:\d+)?(?:\/|$)/i.test(value) + ? `http://${value}` + : /^[a-z][a-z\d+.-]*:/i.test(value) + ? value + : `https://${value}` + if (!URL.canParse(candidate)) throw browserError("invalid_url", "Enter a valid HTTP or HTTPS URL.") + const url = new URL(candidate) + if ((url.protocol !== "http:" && url.protocol !== "https:") || url.username || url.password) { + throw browserError("invalid_url", "Only HTTP, HTTPS, and about:blank URLs are supported.") + } + if (url.href.length > 16_384) throw browserError("invalid_url", "The browser URL is too long.") + return url.href +} +function bounded( + run: () => PromiseLike, + signal: AbortSignal | undefined, + timeout: number, + timeoutMessage: string, +) { + throwIfAborted(signal) + const timedOut = AbortSignal.timeout(timeout) + const abort = signal ? AbortSignal.any([signal, timedOut]) : timedOut + return new Promise((resolve, reject) => { + abort.addEventListener( + "abort", + () => + reject( + timedOut.aborted + ? browserError("timeout", timeoutMessage) + : browserError("aborted", "The browser action was aborted."), + ), + { once: true }, + ) + Promise.resolve().then(run).then(resolve, reject) + }) +} +function staleProtocolError(input: unknown) { + return /Could not find (node|object)|No node with given id|Node with given id does not belong|Could not push node|Could not compute box model|stale element/i.test( + input instanceof Error ? input.message : String(input), + ) +} +function snapshotExpression(nextRef: number) { + return `(() => { + const interactive = new Set(["button","checkbox","combobox","link","menuitem","option","radio","searchbox","slider","spinbutton","switch","tab","textbox"]) + const readable = new Set(["article","cell","columnheader","heading","img","list","listitem","p","region","row","rowheader","table"]) + const roleFor = (element) => { + const explicit = element.getAttribute("role") + if (explicit) return explicit.slice(0, 100).split(/\\s+/)[0] + if (/^H[1-6]$/.test(element.tagName)) return "heading" + if (element.tagName === "INPUT") { + if (element.type === "checkbox") return "checkbox" + if (element.type === "radio") return "radio" + if (element.type === "range") return "slider" + if (element.type === "number") return "spinbutton" + if (element.type === "search") return "searchbox" + return "textbox" + } + return ({A:"link",ARTICLE:"article",BUTTON:"button",IMG:"img",LI:"listitem",OL:"list",P:"p",SELECT:"combobox",TABLE:"table",TD:"cell",TH:"columnheader",TR:"row",TEXTAREA:"textbox",UL:"list"})[element.tagName] || element.tagName.toLowerCase() + } + const clean = (value) => String(value || "").slice(0, 1000).replace(/\\s+/g, " ").trim().slice(0, 300) + const textFor = (element) => { + const queue = Array.from(element.childNodes).slice(0, 20) + const parts = [] + let visited = 0 + while (queue.length && visited++ < 20) { + const item = queue.shift() + if (item.nodeType === Node.TEXT_NODE) parts.push(item.nodeValue || "") + queue.push(...Array.from(item.childNodes).slice(0, Math.max(0, 20 - queue.length - visited))) + } + return parts.join(" ") + } + const nodes = [] + const refs = Object.create(null) + const walker = document.createTreeWalker(document.body || document.documentElement, NodeFilter.SHOW_ELEMENT) + let visited = 0 + let ref = ${Math.max(0, Math.floor(nextRef))} + while (visited++ < ${snapshotLimit}) { + const element = walker.nextNode() + if (!element) break + if (element.hidden || element.getAttribute("aria-hidden") === "true" || (element.tagName === "INPUT" && element.type === "hidden")) continue + const role = clean(roleFor(element)).replace(/[^a-zA-Z0-9_-]/g, "").slice(0, 40) || "node" + const isInteractive = interactive.has(role) || element.tabIndex >= 0 + if (!isInteractive && !readable.has(role)) continue + const editable = ["INPUT","TEXTAREA","SELECT"].includes(element.tagName) || ["textbox","searchbox","combobox","spinbutton"].includes(role) || element.isContentEditable + const labelledBy = element.getAttribute("aria-labelledby") + const label = labelledBy && document.getElementById(labelledBy) + const token = isInteractive ? "e" + (++ref) : undefined + if (token) refs[token] = element + let depth = 0 + for (let item = element.parentElement; item && depth < 6; item = item.parentElement) depth++ + nodes.push({ + token, + role, + name: clean(element.getAttribute("aria-label") || (label && textFor(label)) || element.alt || (editable ? "" : textFor(element))), + value: editable ? "" : clean(element.value), + depth, + checked: "checked" in element ? Boolean(element.checked) : undefined, + disabled: "disabled" in element ? Boolean(element.disabled) : undefined, + expanded: element.getAttribute("aria-expanded") === "true" ? true : element.getAttribute("aria-expanded") === "false" ? false : undefined, + selected: "selected" in element ? Boolean(element.selected) : undefined, + }) + } + return { result: { nodes, nextRef: ref }, refs } + })()` +} +const fillFunction = `function(token) { + const element = this.refs[token] + if (!element || !element.isConnected) throw new Error("stale element") + const role = String(element.getAttribute("role") || "").split(/\\s+/, 1)[0] + const input = element.tagName === "INPUT" && !["button","checkbox","color","file","hidden","image","radio","range","reset","submit"].includes(String(element.type).toLowerCase()) + const editable = input || element.tagName === "TEXTAREA" || element.isContentEditable || ["textbox","searchbox","combobox","spinbutton"].includes(role) + if (!editable || element.disabled || element.readOnly || element.getAttribute("aria-disabled") === "true" || element.getAttribute("aria-readonly") === "true") return false + element.focus() + return true +}` +function record(input: unknown): input is Record { + return typeof input === "object" && input !== null && !Array.isArray(input) +} diff --git a/packages/client/src/node/browser/driver.ts b/packages/client/src/node/browser/driver.ts index a9ed44164c..8a68a8c8f5 100644 --- a/packages/client/src/node/browser/driver.ts +++ b/packages/client/src/node/browser/driver.ts @@ -1,4 +1,5 @@ import type { Browser } from "@opencode-ai/schema/browser" +import { chromiumDriver, type ChromiumDriver, type ChromiumPort } from "./chromium.js" export interface BrowserProxy { readonly url: string readonly host: string @@ -40,6 +41,11 @@ export const BrowserDriver = { define(create: BrowserDriverFactory): BrowserDriver { return create }, + chromium( + create: (context: BrowserDriverContext) => PromiseLike> | ChromiumPort, + ): ChromiumDriver { + return chromiumDriver(create) + }, } export function browserDriverFactory(driver: BrowserDriver) { diff --git a/packages/client/src/node/index.ts b/packages/client/src/node/index.ts index 971e58412d..1829512d61 100644 --- a/packages/client/src/node/index.ts +++ b/packages/client/src/node/index.ts @@ -23,6 +23,7 @@ export type { BrowserDriverInstance, BrowserProxy, } from "./browser/driver.js" +export type { ChromiumController, ChromiumDriver, ChromiumPort } from "./browser/chromium.js" export type { BrowserAttachment, BrowserAttachOptions, diff --git a/packages/client/test/node/chromium.test.ts b/packages/client/test/node/chromium.test.ts new file mode 100644 index 0000000000..f85fd38aba --- /dev/null +++ b/packages/client/test/node/chromium.test.ts @@ -0,0 +1,85 @@ +import { Browser, BrowserDriver, type BrowserDriverContext, type ChromiumPort } from "@opencode-ai/client/node" +import { describe, expect, test } from "bun:test" + +type Port = ChromiumPort<{ readonly name: string }> +type Command = Parameters[0] +type Listener = Parameters[0] + +describe("Chromium browser driver", () => { + test("snapshots accessibility refs and invalidates them with the document generation", async () => { + const port = new FakePort() + const instance = await BrowserDriver.chromium(() => port)({ + proxy: { url: "http://127.0.0.1:1", host: "127.0.0.1", port: 1, credentials: { username: "u", password: "p" } }, + signal: new AbortController().signal, + } satisfies BrowserDriverContext) + const execute = (command: Browser.Command) => instance.execute(command, { signal: new AbortController().signal }) + + const snapshot = await execute({ type: "snapshot", generation: 0 }) + expect(snapshot).toMatchObject({ + type: "snapshot", + content: expect.stringContaining('e1 [button] "Save" disabled=false'), + }) + expect(port.expression).toContain("while (visited++ < 500)") + expect(port.expression).not.toContain("textContent") + await execute({ type: "click", ref: Browser.Ref.make("e1"), generation: 0 }) + + port.emit() + expect(instance.resource.state().generation).toBe(1) + expect(port.commands.some((command) => command.method === "Runtime.releaseObject")).toBe(true) + await expect(execute({ type: "click", ref: Browser.Ref.make("e1"), generation: 1 })).rejects.toMatchObject({ + code: "stale_ref", + }) + await instance.resource.dispose() + }) +}) + +class FakePort implements Port { + readonly resource = { name: "chromium" } + readonly listeners = new Set() + readonly commands: Command[] = [] + current = { url: "https://example.com/", title: "Example", loading: false, canGoBack: false, canGoForward: false } + expression = "" + + state() { + return this.current + } + subscribe(listener: Listener) { + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } + navigate() {} + back() {} + forward() {} + reload() {} + stop() {} + send(command: Command) { + this.commands.push(command) + if (command.method === "Runtime.evaluate") { + this.expression = command.params.expression + return Promise.resolve({ result: { objectId: "snapshot" } }) + } + if (command.method !== "Runtime.callFunctionOn") return Promise.resolve({}) + if (command.params.functionDeclaration === "function() { return this.result }") { + return Promise.resolve({ + result: { + value: { + nodes: [{ token: "e1", role: "button", name: "Save", value: "", depth: 1, disabled: false }], + nextRef: 1, + }, + }, + }) + } + return Promise.resolve({ result: { value: { x: 25, y: 40 } } }) + } + viewport() { + return { width: 800, height: 600 } + } + screenshot() { + return Promise.resolve({ data: new Uint8Array(), width: 800, height: 600 }) + } + dispose() {} + emit() { + this.current = { ...this.current, url: "https://next.example/" } + this.listeners.forEach((listener) => listener({ state: this.current, mainDocumentChanged: true })) + } +} diff --git a/packages/client/test/node/package-smoke.ts b/packages/client/test/node/package-smoke.ts new file mode 100644 index 0000000000..ccafe821e8 --- /dev/null +++ b/packages/client/test/node/package-smoke.ts @@ -0,0 +1,133 @@ +import { expect, test } from "bun:test" +import { mkdir, mkdtemp, rm } from "node:fs/promises" +import { join, relative, resolve } from "node:path" +import { pathToFileURL } from "node:url" + +const directory = resolve(import.meta.dir, "../..") + +test("built Node entrypoint imports and exposes browser registration in Node", async () => { + await buildClient() + const output = await Bun.file(join(directory, "dist/node/index.js")).text() + expect(output).not.toMatch(/(?:from\s+|import\s*)["']\.\.?\//) + + const temporary = await mkdtemp(join(import.meta.dir, ".node-package-")) + try { + await Bun.write(join(temporary, "index.mjs"), output) + await stageWorkspaceDependencies(temporary) + const child = Bun.spawn( + ["node", "--input-type=module", "-e", nodeScenario(pathToFileURL(join(temporary, "index.mjs")).href)], + { cwd: temporary, stdout: "pipe", stderr: "pipe" }, + ) + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + if (exitCode !== 0) throw new Error(stderr || stdout) + expect(stdout.trim()).toBe("ok") + } finally { + await rm(temporary, { recursive: true, force: true }) + } +}, 60_000) + +async function buildClient() { + const child = Bun.spawn([process.execPath, "run", "build"], { + cwd: directory, + stdout: "pipe", + stderr: "pipe", + }) + const [exitCode, stdout, stderr] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + new Response(child.stderr).text(), + ]) + if (exitCode !== 0) throw new Error(stdout + stderr) +} + +async function stageWorkspaceDependencies(temporary: string) { + const schema = join(temporary, "node_modules/@opencode-ai/schema") + const protocol = join(temporary, "node_modules/@opencode-ai/protocol") + await Promise.all([mkdir(schema, { recursive: true }), mkdir(protocol, { recursive: true })]) + + const schemaEntry = join(temporary, "schema.ts") + const protocolEntry = join(temporary, "protocol.ts") + await Promise.all([ + Bun.write( + schemaEntry, + [ + `export { Browser } from ${JSON.stringify(importPath(temporary, resolve(directory, "../schema/src/browser.ts")))}`, + `export { BrowserControl } from ${JSON.stringify(importPath(temporary, resolve(directory, "../schema/src/browser-control.ts")))}`, + `export { BrowserTunnel } from ${JSON.stringify(importPath(temporary, resolve(directory, "../schema/src/browser-tunnel.ts")))}`, + `export { Session } from ${JSON.stringify(importPath(temporary, resolve(directory, "../schema/src/session.ts")))}`, + ].join("\n"), + ), + Bun.write( + protocolEntry, + [ + `export { BrowserControlProtocol } from ${JSON.stringify(importPath(temporary, resolve(directory, "../protocol/src/browser-control.ts")))}`, + `export { BrowserTunnelProtocol } from ${JSON.stringify(importPath(temporary, resolve(directory, "../protocol/src/browser-tunnel.ts")))}`, + ].join("\n"), + ), + ]) + const [schemaBuild, protocolBuild] = await Promise.all([ + Bun.build({ + entrypoints: [schemaEntry], + outdir: schema, + naming: "index.js", + target: "node", + format: "esm", + packages: "bundle", + }), + Bun.build({ + entrypoints: [protocolEntry], + outdir: protocol, + naming: "index.js", + target: "node", + format: "esm", + packages: "bundle", + }), + ]) + if (!schemaBuild.success) throw new Error(schemaBuild.logs.map((log) => log.message).join("\n")) + if (!protocolBuild.success) throw new Error(protocolBuild.logs.map((log) => log.message).join("\n")) + await Promise.all([ + Bun.write( + join(schema, "package.json"), + JSON.stringify({ + type: "module", + exports: { + "./browser": "./index.js", + "./browser-control": "./index.js", + "./browser-tunnel": "./index.js", + "./session": "./index.js", + }, + }), + ), + Bun.write( + join(protocol, "package.json"), + JSON.stringify({ + type: "module", + exports: { + "./browser-control": "./index.js", + "./browser-tunnel": "./index.js", + }, + }), + ), + ]) +} + +function importPath(from: string, to: string) { + const path = relative(from, to).replaceAll("\\", "/") + return path.startsWith(".") ? path : `./${path}` +} + +function nodeScenario(moduleURL: string) { + return `const sdk = await import(${JSON.stringify(moduleURL)}) +if (typeof sdk.OpenCode.make !== "function") throw new Error("Missing OpenCode.make") +if (typeof sdk.BrowserDriver.define !== "function") throw new Error("Missing BrowserDriver.define") +if (typeof sdk.BrowserDriver.chromium !== "function") throw new Error("Missing BrowserDriver.chromium") +if (typeof sdk.BrowserDriverError !== "function") throw new Error("Missing BrowserDriverError") +if (!sdk.Browser.State) throw new Error("Missing canonical Browser export") +const client = sdk.OpenCode.make({ baseUrl: "http://127.0.0.1:1" }) +if (typeof client.browser.register !== "function") throw new Error("Missing browser.register") +console.log("ok")` +} diff --git a/packages/client/test/types/node-consumer.ts b/packages/client/test/types/node-consumer.ts index 2ddc3e1154..8e9a6d7a3c 100644 --- a/packages/client/test/types/node-consumer.ts +++ b/packages/client/test/types/node-consumer.ts @@ -5,6 +5,9 @@ import { OpenCode, type BrowserAttachment, type BrowserRegistration, + type ChromiumController, + type ChromiumDriver, + type ChromiumPort, } from "@opencode-ai/client/node" const state: Browser.State = { @@ -16,7 +19,7 @@ const state: Browser.State = { generation: 0, } -const driver = BrowserDriver.define<{ readonly proxyURL: string }>((context) => ({ +const factory: BrowserDriver<{ readonly proxyURL: string }> = (context) => ({ resource: { proxyURL: context.proxy.url }, state: () => state, subscribe: () => () => undefined, @@ -24,7 +27,10 @@ const driver = BrowserDriver.define<{ readonly proxyURL: string }>((context) => throw new BrowserDriverError(options.signal.aborted ? "aborted" : "internal", "Command unavailable") }, dispose: () => undefined, -})) +}) +const driver = BrowserDriver.define(factory) +declare const port: ChromiumPort<{ readonly page: true }> +const chromium: ChromiumDriver<{ readonly page: true }> = BrowserDriver.chromium(() => port) declare const client: ReturnType const registration: Promise = client.browser.register({ @@ -33,5 +39,9 @@ const registration: Promise = client.browser.register({ }) void registration.then((handle) => { const attachment: Promise> = handle.attach({ driver }) + const chromiumAttachment: Promise>> = handle.attach({ + driver: chromium, + }) void attachment + void chromiumAttachment })