This commit is contained in:
James Long 2026-05-17 21:48:00 -04:00
commit 241b7b5792
13 changed files with 92434 additions and 26 deletions

View file

@ -0,0 +1,19 @@
Simulation script runner.
Usage:
bun run.ts <script.json> [options]
Options:
--mcp <url> MCP endpoint (default http://127.0.0.1:43110/mcp)
--chunk <n> Actions per step batch (default 3)
--max-steps <n> Hard cap on step calls (default unlimited)
--level <lvl> Stop level: DEBUG|INFO|WARN|ERROR (default ERROR)
--message-includes <s> Only stop when message includes substring
--service-includes <s> Only stop when tag.service includes substring
--reset Reset sim state + restart TUI before load
--no-reset Skip reset (default)
--keep-going Don't stop on errors; continue to end
--quiet Suppress per-batch progress
--json Emit JSON summary at the end
--check-every <n> Check logs every N batches (default 1)

File diff suppressed because it is too large Load diff

View file

@ -373,38 +373,133 @@ async function discoverInstances(input: { startPort?: number; maxPorts?: number;
return { instances: instances(), discovered: found, scanned: { startPort, maxPorts, consecutiveFailures } }
}
type ScriptAction = z.infer<typeof ScriptActionSchema>
type ScriptCounts = { uiActions: number; fileWrites: number; llmScriptsQueued: number; waits: number }
async function executeAction(options: Options, action: ScriptAction, counts: ScriptCounts) {
if (action.type === "writeFile") {
await control(options, "POST", "/experimental/simulation/filesystem/write", {
path: action.path,
content: action.content,
})
counts.fileWrites++
return
}
if (action.type === "enqueueLLM") {
await control(options, "POST", "/experimental/simulation/llm/enqueue", { scripts: action.scripts })
counts.llmScriptsQueued += action.scripts.length
return
}
if (action.type === "wait") {
await new Promise((resolve) => setTimeout(resolve, action.ms ?? 1_000))
await current(options).harness.renderOnce()
counts.waits++
return
}
await SimulationActions.execute(current(options).harness, action)
counts.uiActions++
}
async function runScript(options: Options, file: string) {
const parsed = ScriptSchema.parse(await Bun.file(file).json())
const actions = Array.isArray(parsed) ? parsed : parsed.actions
const counts = { uiActions: 0, fileWrites: 0, llmScriptsQueued: 0, waits: 0 }
for (const action of actions) {
if (action.type === "writeFile") {
await control(options, "POST", "/experimental/simulation/filesystem/write", {
path: action.path,
content: action.content,
})
counts.fileWrites++
continue
}
if (action.type === "enqueueLLM") {
await control(options, "POST", "/experimental/simulation/llm/enqueue", { scripts: action.scripts })
counts.llmScriptsQueued += action.scripts.length
continue
}
if (action.type === "wait") {
await new Promise((resolve) => setTimeout(resolve, action.ms ?? 1_000))
await current(options).harness.renderOnce()
counts.waits++
continue
}
await SimulationActions.execute(current(options).harness, action)
counts.uiActions++
}
const counts: ScriptCounts = { uiActions: 0, fileWrites: 0, llmScriptsQueued: 0, waits: 0 }
for (const action of actions) await executeAction(options, action, counts)
return { file, actions: actions.length, ...counts, snapshot: snapshot(options) }
}
// ─── Step-controlled script execution ───────────────────────────────────────
//
// `simulation_script_load` parses a script and stores its actions in process
// memory keyed by a generated id. The script is NOT executed yet. Subsequent
// calls to `simulation_script_step` advance the cursor by one or more actions
// at a time, returning the snapshot and updated counts after each batch. This
// lets MCP clients drive scripts at their own pace and inspect state between
// steps. Only one script is active at a time per process; loading a new one
// while a previous one is still pending requires either consuming it to
// completion, calling `simulation_script_cancel`, or specifying replace=true.
interface LoadedScript {
readonly id: string
readonly file: string | null
readonly actions: ScriptAction[]
cursor: number
readonly counts: ScriptCounts
readonly loadedAt: string
}
let loaded: LoadedScript | undefined
let loadSeq = 0
function loadedSummary(state: LoadedScript) {
return {
id: state.id,
file: state.file,
total: state.actions.length,
cursor: state.cursor,
remaining: state.actions.length - state.cursor,
done: state.cursor >= state.actions.length,
counts: { ...state.counts },
loadedAt: state.loadedAt,
}
}
async function loadScript(input: {
file?: string
script?: unknown
replace?: boolean
}) {
if (loaded && loaded.cursor < loaded.actions.length && !input.replace) {
throw new Error(
`A script is already loaded (id=${loaded.id}, ${loaded.actions.length - loaded.cursor} actions remaining). Pass replace=true or call simulation_script_cancel first.`,
)
}
const raw = input.file
? await Bun.file(input.file).json()
: (input.script ?? (() => {
throw new Error("simulation_script_load requires either `file` or `script`.")
})())
const parsed = ScriptSchema.parse(raw)
const actions = Array.isArray(parsed) ? parsed : parsed.actions
loaded = {
id: `script-${(++loadSeq).toString(36)}`,
file: input.file ?? null,
actions: [...actions] as ScriptAction[],
cursor: 0,
counts: { uiActions: 0, fileWrites: 0, llmScriptsQueued: 0, waits: 0 },
loadedAt: new Date().toISOString(),
}
return loadedSummary(loaded)
}
async function stepScript(options: Options, input: { steps?: number; renderEach?: boolean }) {
if (!loaded) throw new Error("No script loaded. Call simulation_script_load first.")
const max = input.steps ?? 1
const executed: ScriptAction[] = []
for (let i = 0; i < max && loaded.cursor < loaded.actions.length; i++) {
const action = loaded.actions[loaded.cursor]!
executed.push(action)
await executeAction(options, action, loaded.counts)
loaded.cursor++
if (input.renderEach && i < max - 1) await current(options).harness.renderOnce()
}
return {
executed,
state: loadedSummary(loaded),
snapshot: snapshot(options),
}
}
function cancelScript() {
const was = loaded ? loadedSummary(loaded) : null
loaded = undefined
return { cancelled: was !== null, was }
}
function statusScript() {
return loaded ? loadedSummary(loaded) : null
}
async function runOnTargets<A>(
options: Options,
target: z.infer<typeof TargetSchema>,
@ -504,6 +599,48 @@ function createServer(options: Options) {
async (input) => toolResult(await runScript(options, input.path)),
)
server.registerTool(
"simulation_script_load",
{
description:
"Load a simulation script into memory WITHOUT executing it. Pass `path` to load from a JSON file on disk, or `script` to load inline JSON. Returns the parsed action count and a script id. Use `simulation_script_step` to execute actions one (or N) at a time. Only one script may be loaded at once unless `replace` is true.",
inputSchema: z.object({
path: z.string().optional(),
script: z.any().optional(),
replace: z.boolean().optional(),
}),
},
async (input) => toolResult(await loadScript({ file: input.path, script: input.script, replace: input.replace })),
)
server.registerTool(
"simulation_script_step",
{
description:
"Execute the next action(s) of the loaded script and return the snapshot afterwards. Defaults to one step. Pass `steps` to advance multiple actions in a single call (1-100). When `renderEach` is true, the simulated TUI is forced to render between steps.",
inputSchema: z.object({
steps: z.number().int().min(1).max(100).optional(),
renderEach: z.boolean().optional(),
}),
},
async (input) => toolResult(await stepScript(options, { steps: input.steps, renderEach: input.renderEach })),
)
server.registerTool(
"simulation_script_status",
{
description:
"Return the currently-loaded script's progress: id, total actions, cursor, remaining, and execution counts. Returns null when nothing is loaded.",
},
async () => toolResult({ status: statusScript() }),
)
server.registerTool(
"simulation_script_cancel",
{ description: "Discard the currently-loaded script (if any). Subsequent step calls fail until a new script is loaded." },
async () => toolResult(cancelScript()),
)
if ("runtime" in options) {
server.registerTool("simulation_restart", { description: "Restart the simulated TUI and backend while keeping MCP alive." }, async () =>
toolResult(await options.runtime.restart()),
@ -582,6 +719,45 @@ function createServer(options: Options) {
},
)
server.registerTool(
"simulation_log_get",
{
description:
"Return the in-memory log buffer captured by `@opencode-ai/core/util/log` inside the simulated backend (capped at 5000 most recent entries). Filter by `level` to return only entries at or above that level. Also supports optional `limit` and substring filters.",
inputSchema: z.object({
level: z.enum(["DEBUG", "INFO", "WARN", "ERROR"]).optional(),
limit: z.number().int().min(1).max(5000).optional(),
messageIncludes: z.string().optional(),
serviceIncludes: z.string().optional(),
}),
},
async (input) => {
const data = await control(options, "GET", "/experimental/simulation/log/entries")
let entries = (data?.entries ?? []) as Array<{
time: string
level: "DEBUG" | "INFO" | "WARN" | "ERROR"
tags: Record<string, unknown>
message: string
}>
if (input.level) {
const priority = { DEBUG: 0, INFO: 1, WARN: 2, ERROR: 3 } as const
const threshold = priority[input.level]
entries = entries.filter((e) => priority[e.level] >= threshold)
}
if (input.messageIncludes) entries = entries.filter((e) => e.message.includes(input.messageIncludes!))
if (input.serviceIncludes)
entries = entries.filter((e) => String(e.tags?.service ?? "").includes(input.serviceIncludes!))
if (typeof input.limit === "number") entries = entries.slice(-input.limit)
return toolResult({ entries, total: entries.length })
},
)
server.registerTool(
"simulation_log_clear",
{ description: "Clear the simulated backend's in-memory log buffer." },
async () => toolResult(await control(options, "POST", "/experimental/simulation/log/clear")),
)
if (masterEnabled()) {
server.registerTool(
"simulation_instances_discover",

View file

@ -27,7 +27,6 @@ if (process.env.OPENCODE_SIMULATION_CWD) {
Global.Path.state = `${process.env.OPENCODE_SIMULATION_CWD}/.local/state/opencode`
Global.Path.tmp = `${process.env.OPENCODE_SIMULATION_CWD}/tmp/opencode`
Global.Path.bin = `${Global.Path.cache}/bin`
Global.Path.log = `${Global.Path.data}/log`
Global.Path.repos = `${Global.Path.data}/repos`
Object.defineProperty(process, "cwd", {
value: () => process.env.OPENCODE_SIMULATION_CWD!,

View file

@ -1,6 +1,7 @@
import { Simulation } from "@/testing/simulation/service"
import { Effect } from "effect"
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"
import * as Log from "@opencode-ai/core/util/log"
const ok = { ok: true }
@ -56,6 +57,19 @@ export const simulationRoute = HttpRouter.use((router) =>
)
yield* router.add("GET", "/experimental/simulation/snapshot", () => json(simulation.snapshot()))
yield* router.add("GET", "/experimental/simulation/log/entries", () =>
Effect.succeed(HttpServerResponse.jsonUnsafe({ entries: Log.entries() })),
)
yield* router.add("POST", "/experimental/simulation/log/clear", () =>
Effect.succeed(
HttpServerResponse.jsonUnsafe(((): { cleared: true } => {
Log.clearEntries()
return { cleared: true }
})()),
),
)
}),
)

View file

@ -161,9 +161,21 @@ export const SimulatedTypescript: LSPServer.Info = {
// backend reports clients for the same files. The simulated root is just
// the instance directory — we don't probe the filesystem here because the
// simulated FS is in-memory and the only "project" is `/opencode`.
extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"],
extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts", ".json", ".md"],
root: async (_file, ctx) => ctx.directory,
async spawn(_root, _ctx) {
// Real LSP server spawn is an actual `child_process.spawn` that crosses an
// I/O boundary and forces the surrounding `await` to truly yield to the
// microtask queue (and beyond). The simulated path is otherwise fully
// synchronous, which lets the calling Effect fiber's context survive
// `await server.spawn(...)` in cases where production would lose it.
//
// To make the simulation a faithful behavioral surrogate of the real
// server (and surface bugs like #27880), force a real async yield here
// via setTimeout so the awaiting fiber is fully torn down before we
// resume.
await new Promise<void>((resolve) => setTimeout(resolve, 500))
const { process, serverInput, serverOutput } = createFakeProcess()
const connection = createMessageConnection(

View file

@ -4,6 +4,7 @@
{ "type": "pressKey", "key": "b" },
{ "type": "writeFile", "path": "src/greeting.ts", "content": "export function greet(name: string) {\n return `Hi, ${name}`\n}\n" },
{
"_comment": "Three scripts enqueued: (1) main session prompt — long explanation + apply_patch tool call; (2) post-tool-result follow-up text; (3) title-generation small-model call. Title gen fires in parallel with the main call, so script order is by FIFO race winner. With three scripts queued the post-tool follow-up always finds a real script instead of falling back to the default 'Simulation mock response.'",
"type": "enqueueLLM",
"scripts": [
{
@ -35,6 +36,15 @@
],
"usage": { "inputTokens": 140, "outputTokens": 16, "totalTokens": 156 },
"finish": "stop"
},
{
"steps": [
[
{ "type": "text", "content": "Friendlier greeting" }
]
],
"usage": { "inputTokens": 20, "outputTokens": 4, "totalTokens": 24 },
"finish": "stop"
}
]
},

View file

@ -0,0 +1,578 @@
#!/usr/bin/env bun
/**
* Generator for `09_diverse_llm_responses.json`.
*
* Produces a single simulation script that queues 1000 LLM responses with
* maximum diversity across:
* - finish reasons (stop, tool-calls, error, length, unknown)
* - step shapes (single text, multi-text, thinking + text, text + tool-call,
* multiple tool-calls, thinking-only, error-only)
* - tool variety: edit, write, apply_patch, read, grep, glob, bash, todowrite,
* webfetch, websearch, lsp, task, task_status, plan, question, skill,
* repo_clone, repo_overview, invalid
* - parameter shapes per tool (varied filePaths, patterns, commands, queries)
*
* The generator is deterministic (seeded RNG) so the same JSON is regenerated.
*
* Run with:
* bun test/testing/simulation/scripts/09_diverse_llm_responses.gen.ts
*
* Output: `09_diverse_llm_responses.json` (next to this file).
*/
import { writeFileSync } from "fs"
import path from "path"
const TOTAL = 1000
const SEED = 0x09abcdef
const OUTPUT_PATH = path.join(import.meta.dirname, "09_diverse_llm_responses.json")
// ─── Seeded RNG (mulberry32) ─────────────────────────────────────────────────
function mulberry32(seed: number) {
let state = seed >>> 0
return () => {
state = (state + 0x6d2b79f5) >>> 0
let t = state
t = Math.imul(t ^ (t >>> 15), t | 1)
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
}
}
const rand = mulberry32(SEED)
const pick = <T>(items: readonly T[]): T => items[Math.floor(rand() * items.length)]!
const int = (min: number, max: number) => min + Math.floor(rand() * (max - min + 1))
const maybe = (p: number) => rand() < p
// ─── Vocab ───────────────────────────────────────────────────────────────────
const FILE_NAMES = [
"src/index.ts",
"src/server.ts",
"src/util/log.ts",
"src/util/string.ts",
"src/feature/auth.ts",
"src/feature/cart.ts",
"src/feature/checkout.tsx",
"src/components/Button.tsx",
"src/components/Modal.tsx",
"src/lib/db.ts",
"src/lib/cache.ts",
"src/api/users.ts",
"src/api/orders.ts",
"test/index.test.ts",
"test/auth.test.ts",
"test/cart.test.ts",
"scripts/build.ts",
"scripts/deploy.sh",
"config/eslint.json",
"config/tsconfig.json",
"package.json",
"README.md",
"CHANGELOG.md",
"docs/api.md",
"docs/getting-started.md",
"Dockerfile",
".github/workflows/ci.yml",
]
const DIR_NAMES = ["src", "src/feature", "src/util", "test", "scripts", "config", "docs"]
const PATTERNS = [
"TODO",
"FIXME",
"console\\.log",
"function\\s+\\w+",
"import .* from",
"export const",
"throw new Error",
"async\\s+function",
"class\\s+\\w+",
"interface\\s+\\w+",
]
const GLOBS = [
"**/*.ts",
"**/*.tsx",
"src/**/*.ts",
"test/**/*.test.ts",
"**/*.{ts,tsx}",
"**/*.md",
"**/*.json",
"scripts/*.sh",
]
const COMMANDS = [
"ls -la",
"pwd",
"cat README.md",
"wc -l src/index.ts",
"git status",
"git log --oneline -5",
"git diff --stat",
"bun install",
"bun test",
"bun run build",
"npm run lint",
"rg TODO",
"find . -name '*.ts' -newer package.json",
"echo 'hello'",
"date",
]
const WEB_URLS = [
"https://example.com/api/data",
"https://docs.opencode.ai/configuration",
"https://github.com/anomalyco/opencode",
"https://api.openai.com/v1/models",
"https://registry.npmjs.org/effect",
"https://nodejs.org/api/fs.html",
"https://typescript.org/docs/handbook",
]
const SEARCH_QUERIES = [
"rust async error handling",
"effect-ts schema validation",
"react server components",
"typescript discriminated unions",
"bun sqlite performance",
"lsp protocol initialize",
"git rebase squash workflow",
]
const SKILLS = ["customize-opencode", "effect", "improve-codebase-architecture", "gmail"]
const SUBAGENTS = ["explore", "general"]
const PLAIN_TEXT = [
"Looking at the code now.",
"Let me inspect the relevant files.",
"I'll start by reading the entrypoint.",
"Checking the test suite for related coverage.",
"Tracing the call chain through the layer composition.",
"This looks like a missing dependency in the layer graph.",
"I'll add a small helper to factor out the duplication.",
"Renaming the symbol everywhere it's used.",
"Bumping the version in package.json.",
"Adding a changelog entry.",
"Running the formatter.",
"Re-running the typecheck.",
"All clean — no type errors.",
"Tests pass locally.",
"Drafting the PR description.",
]
const THINKING = [
"Need to figure out which layer is missing the dependency.",
"The error stack points at instance-state.ts — likely missing InstanceRef.",
"Best to check if the cache is being invalidated correctly.",
"Tradeoff: inline vs extract helper. Inline is shorter.",
"Race condition seems likely given the await boundary.",
"Looking at the diff to spot the regression.",
"Need to make sure the test asserts the post-condition.",
"Let me re-read the spec to be sure.",
]
const FINISH_REASONS: ("stop" | "tool-calls" | "length" | "unknown")[] = [
"stop",
"tool-calls",
"stop",
"tool-calls",
"stop",
"length",
"stop",
"unknown",
]
// ─── Tool-call generators ────────────────────────────────────────────────────
type ToolCall = {
type: "tool-call"
toolCallId: string
toolName: string
input: Record<string, unknown>
}
let toolCallSeq = 0
const nextToolCallId = () => `tc-${(++toolCallSeq).toString(36)}`
const TOOL_GENERATORS: ReadonlyArray<() => ToolCall> = [
// edit
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "edit",
input: {
filePath: `/opencode/${pick(FILE_NAMES)}`,
oldString: pick(["return null", "// TODO", "const x = 1", "throw new Error(\"!\")", "if (true)"]),
newString: pick(["return undefined", "// fixed", "const x = 2", "throw new Error(\"unexpected\")", "if (cond)"]),
...(maybe(0.2) ? { replaceAll: true } : {}),
},
}),
// write
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "write",
input: {
filePath: `/opencode/${pick(FILE_NAMES)}`,
content: pick([
"export const value = 42\n",
"// generated\nexport default {}\n",
"TODO: fill in\n",
'{\n "version": "0.0.1"\n}\n',
]),
},
}),
// apply_patch
() => {
const file = `/opencode/${pick(FILE_NAMES)}`
const before = pick(["return null", "const x = 1", "// old"])
const after = pick(["return undefined", "const x = 2", "// new"])
return {
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "apply_patch",
input: {
patchText: `*** Begin Patch\n*** Update File: ${file}\n@@\n- ${before}\n+ ${after}\n*** End Patch\n`,
},
}
},
// read
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "read",
input: {
filePath: `/opencode/${pick(FILE_NAMES)}`,
...(maybe(0.3) ? { offset: int(1, 50) } : {}),
...(maybe(0.3) ? { limit: int(10, 200) } : {}),
},
}),
// grep
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "grep",
input: {
pattern: pick(PATTERNS),
...(maybe(0.5) ? { path: pick(DIR_NAMES) } : {}),
...(maybe(0.5) ? { include: pick(["*.ts", "*.tsx", "*.{ts,tsx}", "*.md", "*.json"]) } : {}),
},
}),
// glob
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "glob",
input: {
pattern: pick(GLOBS),
...(maybe(0.3) ? { path: pick(DIR_NAMES) } : {}),
},
}),
// bash
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "bash",
input: {
command: pick(COMMANDS),
description: pick(["List files", "Show status", "Print working dir", "Run tests", "Lint code"]),
...(maybe(0.2) ? { timeout: int(1000, 60000) } : {}),
},
}),
// todowrite
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "todowrite",
input: {
todos: Array.from({ length: int(1, 4) }, () => ({
content: pick([
"Investigate failing test",
"Refactor layer composition",
"Add typecheck step",
"Update docs",
"Bump dependencies",
]),
status: pick(["pending", "in_progress", "completed", "cancelled"]),
priority: pick(["high", "medium", "low"]),
})),
},
}),
// webfetch
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "webfetch",
input: {
url: pick(WEB_URLS),
...(maybe(0.4) ? { format: pick(["markdown", "text", "html"]) } : {}),
...(maybe(0.2) ? { timeout: int(5, 60) } : {}),
},
}),
// websearch
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "websearch",
input: {
query: pick(SEARCH_QUERIES),
...(maybe(0.3) ? { numResults: int(1, 10) } : {}),
...(maybe(0.3) ? { type: pick(["auto", "fast", "deep"]) } : {}),
},
}),
// lsp
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "lsp",
input: {
file: `/opencode/${pick(FILE_NAMES)}`,
action: pick(["definition", "references", "hover", "documentSymbol", "implementation"]),
...(maybe(0.5) ? { position: { line: int(0, 100), character: int(0, 80) } } : {}),
},
}),
// task
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "task",
input: {
description: pick(["explore feature", "audit dependency graph", "find usage"]),
prompt: pick([
"Look through src/ and summarize the entry points.",
"Find all call sites for `Bus.publish` and explain what they publish.",
]),
subagent_type: pick(SUBAGENTS),
},
}),
// task_status
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "task_status",
input: {
task_id: `task-${int(1, 50).toString(36)}`,
},
}),
// plan
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "plan",
input: {
summary: pick([
"Refactor the layer graph to remove cycles",
"Add diagnostic logging then propose a fix",
"Extract a helper and add tests",
]),
},
}),
// question
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "question",
input: {
questions: [
{
question: pick(["Which directory should I scaffold under?", "Approve the rename?"]),
header: pick(["Pick a directory", "Approve rename"]),
options: [
{ label: "Yes", description: "Approve" },
{ label: "No", description: "Decline" },
],
},
],
},
}),
// skill
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "skill",
input: { name: pick(SKILLS) },
}),
// repo_clone
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "repo_clone",
input: {
url: pick(["https://github.com/anomalyco/opencode", "https://github.com/effect-ts/effect"]),
...(maybe(0.4) ? { ref: pick(["main", "dev", "v1.0.0"]) } : {}),
},
}),
// repo_overview
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "repo_overview",
input: {
...(maybe(0.5) ? { path: pick(DIR_NAMES) } : {}),
...(maybe(0.3) ? { depth: int(1, 4) } : {}),
},
}),
// invalid (sanity / fuzz)
() => ({
type: "tool-call",
toolCallId: nextToolCallId(),
toolName: "invalid",
input: {
tool: pick(["foo", "bar", "definitely_not_a_tool"]),
error: pick(["unknown tool", "missing argument"]),
},
}),
]
// ─── Step-shape generators ───────────────────────────────────────────────────
type Step = ReadonlyArray<
| { type: "text"; content: string }
| { type: "thinking"; content: string }
| ToolCall
>
const STEP_SHAPES: ReadonlyArray<() => Step> = [
// shape 0: single text
() => [{ type: "text", content: pick(PLAIN_TEXT) }],
// shape 1: multi text
() =>
Array.from({ length: int(2, 5) }, () => ({
type: "text" as const,
content: pick(PLAIN_TEXT),
})),
// shape 2: thinking + text
() => [
{ type: "thinking" as const, content: pick(THINKING) },
{ type: "text" as const, content: pick(PLAIN_TEXT) },
],
// shape 3: text + single tool-call
() => [{ type: "text" as const, content: pick(PLAIN_TEXT) }, pick(TOOL_GENERATORS)()],
// shape 4: thinking + text + tool-call
() => [
{ type: "thinking" as const, content: pick(THINKING) },
{ type: "text" as const, content: pick(PLAIN_TEXT) },
pick(TOOL_GENERATORS)(),
],
// shape 5: multiple tool calls in one step
() => [
{ type: "text" as const, content: pick(PLAIN_TEXT) },
...Array.from({ length: int(2, 4) }, () => pick(TOOL_GENERATORS)()),
],
// shape 6: thinking only
() => [{ type: "thinking" as const, content: pick(THINKING) }],
// shape 7: long narrative (5-8 texts)
() =>
Array.from({ length: int(5, 8) }, () => ({
type: "text" as const,
content: pick(PLAIN_TEXT),
})),
// shape 8: solo tool-call (no preamble)
() => [pick(TOOL_GENERATORS)()],
]
// ─── Script generator ────────────────────────────────────────────────────────
type LLMScript = {
steps: Step[]
usage?: { inputTokens: number; outputTokens: number; totalTokens: number }
finish?: "stop" | "tool-calls" | "length" | "unknown"
}
function makeScript(): LLMScript {
const step = pick(STEP_SHAPES)()
const hasToolCall = step.some((item) => item.type === "tool-call")
const finish: LLMScript["finish"] = hasToolCall ? "tool-calls" : pick(FINISH_REASONS)
const inputTokens = int(20, 600)
const outputTokens = int(4, 250)
return {
steps: [step],
usage: { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens },
finish,
}
}
const scripts: LLMScript[] = Array.from({ length: TOTAL }, makeScript)
// ─── Output ──────────────────────────────────────────────────────────────────
// ─── User-driven consumption loop ────────────────────────────────────────────
//
// `enqueueLLM` only fills the backend queue. To actually consume responses we
// need user messages that drive `doStream` calls. Each user turn typically
// consumes 1-2 scripts (one for the main response, one if a tool-call comes
// back and the loop fetches a follow-up; title-gen on the first turn eats one
// more). We send TURNS user messages so the total consumed roughly matches
// the queue size; the default "Simulation mock response." catches any
// remainder.
const PROMPTS = [
"Walk me through the project layout briefly.",
"Find any TODOs in the codebase.",
"Refactor the small helper in src/util/log.ts.",
"Open src/index.ts and explain the entrypoint.",
"Search for `console.log` usage.",
"Run the test suite.",
"Patch the greeting in src/greeting.ts to say Hello.",
"Write a small note in docs/getting-started.md.",
"Show me recent git activity.",
"What's the LSP status?",
"Plan a fix for the failing test.",
"Summarize the changes so far.",
"Look up the npm registry entry for `effect`.",
"Search the web for `lsp protocol initialize`.",
"Outline the next refactor step.",
]
const TURNS = Math.ceil(TOTAL / 2) // ~2 LLM calls per turn on average
const userActions = Array.from({ length: TURNS }, (_, i) => [
{ type: "typeText", text: PROMPTS[i % PROMPTS.length]! },
{ type: "pressEnter" },
// Small wait so the prompt loop can drain LLM calls before the next input.
{ type: "wait", ms: 60 },
]).flat()
const script = {
_comment: `Generated by 09_diverse_llm_responses.gen.ts. ${TOTAL} LLM responses with diverse step shapes, tool calls (every registered tool kind), and finish reasons. Followed by ${TURNS} user turns that drive the backend to consume them. Seed: 0x${SEED.toString(16)}. Re-run the generator to regenerate.`,
actions: [
// Enter Build mode (skip default plan agent).
{ type: "pressKey", key: "x", modifiers: { ctrl: true } },
{ type: "pressKey", key: "b" },
{ type: "enqueueLLM", scripts },
...userActions,
],
}
writeFileSync(OUTPUT_PATH, JSON.stringify(script, null, 2) + "\n")
// ─── Summary printed for the developer ───────────────────────────────────────
const byFinish = scripts.reduce(
(acc, s) => {
const k = s.finish ?? "stop"
acc[k] = (acc[k] ?? 0) + 1
return acc
},
{} as Record<string, number>,
)
const byTool: Record<string, number> = {}
let totalToolCalls = 0
for (const s of scripts) {
for (const step of s.steps) {
for (const item of step) {
if (item.type === "tool-call") {
byTool[item.toolName] = (byTool[item.toolName] ?? 0) + 1
totalToolCalls++
}
}
}
}
console.log(`Wrote ${OUTPUT_PATH}`)
console.log(`Scripts: ${scripts.length}`)
console.log(`Finish reasons:`, byFinish)
console.log(`Tool calls (${totalToolCalls} total):`, byTool)

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,888 @@
#!/usr/bin/env bun
/**
* Simulation script generator.
*
* Builds a single simulation script (JSON) that:
* 1. Enters Build mode (ctrl+x, b).
* 2. Optionally seeds the simulated filesystem with a starter set of files.
* 3. Queues N LLM responses chosen by step-shape and tool-kind weights.
* 4. Drives the TUI through M user turns to consume those LLM responses.
*
* Critically: a stateful FS model tracks which paths exist at each step, so
* `apply_patch`/`edit`/`read` tool calls only ever target real files. `write`
* tool calls add new files into the model so later patches can target them.
*
* Usage:
* bun test/testing/simulation/scripts/generate.ts [options]
*
* Options:
* --out <path> Output JSON path. Default: ./generated.json
* --total <n> Total LLM scripts to queue. Default: 1000
* --turns <n> Number of user turns. Default: ceil(total / 2)
* --seed <n> RNG seed (hex or decimal). Default: 0x09abcdef
* --tools <list> Comma-separated tool kinds to include. Default: all.
* --shapes <list> Comma-separated step shapes. Default: all.
* --weight tool=<n> Override weight for a single tool. Repeatable.
* --weight shape=<n> Override weight for a single shape. Repeatable.
* --seed-files Pre-seed common files into the simulated FS.
* Default: true. Pass --no-seed-files to disable.
* --enable-titles Enqueue +1 short script per turn for title gen.
* Default: true. Pass --no-enable-titles to disable.
*
* Examples:
*
* # Default: 1000 scripts, 500 turns, balanced tool mix
* bun generate.ts --out diverse.json
*
* # Patch-heavy run: only apply_patch + read, 200 turns
* bun generate.ts --total 600 --turns 200 \
* --tools apply_patch,read --out patches.json
*
* # Bias towards bash + apply_patch
* bun generate.ts --weight tool.apply_patch=10 --weight tool.bash=10
*/
import { writeFileSync } from "fs"
import path from "path"
// ─── CLI ─────────────────────────────────────────────────────────────────────
interface CliOptions {
out: string
total: number
turns: number
seed: number
tools: Set<string> | null
shapes: Set<string> | null
toolWeights: Record<string, number>
shapeWeights: Record<string, number>
seedFiles: boolean
enableTitles: boolean
}
function parseArgs(argv: string[]): CliOptions {
const out: CliOptions = {
out: "./generated.json",
total: 1000,
turns: -1,
seed: 0x09abcdef,
tools: null,
shapes: null,
toolWeights: {},
shapeWeights: {},
seedFiles: true,
enableTitles: true,
}
for (let i = 0; i < argv.length; i++) {
const a = argv[i]!
const next = () => {
const v = argv[++i]
if (v === undefined) throw new Error(`${a} requires a value`)
return v
}
switch (a) {
case "--out":
out.out = next()
break
case "--total":
out.total = Number(next())
break
case "--turns":
out.turns = Number(next())
break
case "--seed": {
const v = next()
out.seed = v.startsWith("0x") ? parseInt(v, 16) : Number(v)
break
}
case "--tools":
out.tools = new Set(
next()
.split(",")
.map((s) => s.trim())
.filter(Boolean),
)
break
case "--shapes":
out.shapes = new Set(
next()
.split(",")
.map((s) => s.trim())
.filter(Boolean),
)
break
case "--weight": {
// `tool.apply_patch=5` or `shape.toolCall=3`
const value = next()
const eq = value.indexOf("=")
if (eq < 0) throw new Error(`--weight expects key=number, got ${value}`)
const [k, v] = [value.slice(0, eq), Number(value.slice(eq + 1))]
if (k.startsWith("tool.")) out.toolWeights[k.slice(5)] = v
else if (k.startsWith("shape.")) out.shapeWeights[k.slice(6)] = v
else throw new Error(`--weight key must start with tool. or shape., got ${k}`)
break
}
case "--seed-files":
out.seedFiles = true
break
case "--no-seed-files":
out.seedFiles = false
break
case "--enable-titles":
out.enableTitles = true
break
case "--no-enable-titles":
out.enableTitles = false
break
case "--help":
case "-h":
printHelp()
process.exit(0)
default:
throw new Error(`Unknown argument: ${a}`)
}
}
if (out.turns < 0) out.turns = Math.ceil(out.total / 2)
return out
}
function printHelp() {
console.log(`Simulation script generator.
Usage:
bun generate.ts [options]
Options:
--out <path> Output JSON path (default ./generated.json)
--total <n> Total LLM scripts to queue (default 1000)
--turns <n> User turns (default ceil(total/2))
--seed <n> RNG seed (hex or decimal, default 0x09abcdef)
--tools <list> Comma-separated tool kinds to include
--shapes <list> Comma-separated step shapes
--weight tool.<id>=<n> Override weight for a tool
--weight shape.<id>=<n> Override weight for a step shape
--seed-files Pre-seed starter files (default on)
--no-seed-files Skip pre-seeding files
--enable-titles Pad queue for title-gen calls (default on)
--no-enable-titles Don't pad for title-gen
Available tool kinds:
apply_patch, edit, write, read, grep, glob, bash, todowrite, webfetch,
websearch, lsp, task, task_status, plan, question, skill, repo_clone,
repo_overview, invalid
Available step shapes:
text, multiText, thinkText, textToolCall, thinkTextToolCall, multiToolCall,
thinkOnly, longNarrative, soloToolCall
`)
}
// ─── Seeded RNG (mulberry32) ─────────────────────────────────────────────────
function mulberry32(seed: number) {
let state = seed >>> 0
return () => {
state = (state + 0x6d2b79f5) >>> 0
let t = state
t = Math.imul(t ^ (t >>> 15), t | 1)
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
}
}
// ─── Filesystem model ────────────────────────────────────────────────────────
//
// Tracks the file paths that we've materialized so far. `apply_patch`/`edit`/
// `read` only target paths in this set. `write` adds to the set. The pre-seed
// step populates the initial set so the first random scripts have something to
// patch.
class FsModel {
private paths = new Set<string>()
private seeded: { path: string; content: string }[] = []
seed(path: string, content: string) {
this.paths.add(path)
this.seeded.push({ path, content })
}
add(path: string) {
this.paths.add(path)
}
has(path: string) {
return this.paths.has(path)
}
all(): string[] {
return [...this.paths]
}
seededWrites() {
return this.seeded
}
}
// ─── Vocab ───────────────────────────────────────────────────────────────────
const SEED_FILES: { path: string; content: string }[] = [
{ path: "src/index.ts", content: "export function main() {\n return 0\n}\n" },
{ path: "src/server.ts", content: "export const server = {\n start() {},\n}\n" },
{ path: "src/util/log.ts", content: "export function log(msg: string) {\n console.log(msg)\n}\n" },
{ path: "src/util/string.ts", content: "export const upper = (s: string) => s.toUpperCase()\n" },
{ path: "src/feature/auth.ts", content: "export const auth = { user: null }\n" },
{ path: "src/feature/cart.ts", content: "export const cart: string[] = []\n" },
{ path: "src/lib/db.ts", content: "export const db = { query() { return null } }\n" },
{ path: "src/lib/cache.ts", content: "export const cache = new Map<string, unknown>()\n" },
{ path: "src/api/users.ts", content: "export const users = []\n" },
{ path: "test/index.test.ts", content: "import { test } from 'bun:test'\ntest('ok', () => {})\n" },
{ path: "test/auth.test.ts", content: "import { test } from 'bun:test'\ntest('auth', () => {})\n" },
{ path: "README.md", content: "# project\n\nDescription.\n" },
{ path: "docs/api.md", content: "# api\n\nDocs.\n" },
{ path: "docs/getting-started.md", content: "# getting started\n\nWelcome.\n" },
{ path: "package.json", content: '{\n "name": "project",\n "version": "0.1.0"\n}\n' },
]
const NEW_FILE_CANDIDATES = [
"src/feature/checkout.tsx",
"src/components/Button.tsx",
"src/components/Modal.tsx",
"src/api/orders.ts",
"test/cart.test.ts",
"scripts/build.ts",
"config/eslint.json",
"CHANGELOG.md",
"src/lib/format.ts",
"src/lib/clock.ts",
]
const PATTERNS = [
"TODO",
"FIXME",
"console\\.log",
"function\\s+\\w+",
"import .* from",
"export const",
"throw new Error",
"async\\s+function",
"class\\s+\\w+",
"interface\\s+\\w+",
]
const GLOBS = ["**/*.ts", "**/*.tsx", "src/**/*.ts", "test/**/*.test.ts", "**/*.{ts,tsx}", "**/*.md", "**/*.json"]
const COMMANDS = [
"ls -la",
"pwd",
"cat README.md",
"wc -l src/index.ts",
"git status",
"git log --oneline -5",
"git diff --stat",
"bun install",
"bun test",
"bun run build",
"rg TODO",
"echo 'hello'",
"date",
]
const WEB_URLS = [
"https://example.com/api/data",
"https://docs.opencode.ai/configuration",
"https://github.com/anomalyco/opencode",
"https://api.openai.com/v1/models",
"https://registry.npmjs.org/effect",
"https://nodejs.org/api/fs.html",
]
const SEARCH_QUERIES = [
"rust async error handling",
"effect-ts schema validation",
"react server components",
"typescript discriminated unions",
"bun sqlite performance",
"lsp protocol initialize",
"git rebase squash workflow",
]
const SKILLS = ["customize-opencode", "effect", "improve-codebase-architecture", "gmail"]
const SUBAGENTS = ["explore", "general"]
const PLAIN_TEXT = [
"Looking at the code now.",
"Let me inspect the relevant files.",
"I'll start by reading the entrypoint.",
"Checking the test suite for related coverage.",
"Tracing the call chain through the layer composition.",
"This looks like a missing dependency in the layer graph.",
"I'll add a small helper to factor out the duplication.",
"Renaming the symbol everywhere it's used.",
"Bumping the version in package.json.",
"Adding a changelog entry.",
"Running the formatter.",
"Re-running the typecheck.",
"All clean — no type errors.",
"Tests pass locally.",
"Drafting the PR description.",
]
const THINKING = [
"Need to figure out which layer is missing the dependency.",
"The error stack points at instance-state.ts — likely missing InstanceRef.",
"Best to check if the cache is being invalidated correctly.",
"Tradeoff: inline vs extract helper. Inline is shorter.",
"Race condition seems likely given the await boundary.",
"Looking at the diff to spot the regression.",
"Need to make sure the test asserts the post-condition.",
]
const FINISH_REASONS: ("stop" | "tool-calls" | "length" | "unknown")[] = [
"stop",
"tool-calls",
"stop",
"tool-calls",
"stop",
"length",
"stop",
"unknown",
]
const PROMPTS = [
"Walk me through the project layout briefly.",
"Find any TODOs in the codebase.",
"Refactor the small helper in src/util/log.ts.",
"Open src/index.ts and explain the entrypoint.",
"Search for `console.log` usage.",
"Run the test suite.",
"Patch the greeting in src/greeting.ts to say Hello.",
"Write a small note in docs/getting-started.md.",
"Show me recent git activity.",
"What's the LSP status?",
"Plan a fix for the failing test.",
"Summarize the changes so far.",
"Look up the npm registry entry for `effect`.",
"Search the web for `lsp protocol initialize`.",
"Outline the next refactor step.",
]
// ─── Tool generators ─────────────────────────────────────────────────────────
type ToolCall = {
type: "tool-call"
toolCallId: string
toolName: string
input: Record<string, unknown>
}
interface Helpers {
rand: () => number
pick: <T>(items: readonly T[]) => T
int: (min: number, max: number) => number
maybe: (p: number) => boolean
nextToolCallId: () => string
fs: FsModel
}
const TOOL_KINDS = [
"apply_patch",
"edit",
"write",
"read",
"grep",
"glob",
"bash",
"todowrite",
"webfetch",
"websearch",
"lsp",
"task",
"task_status",
"plan",
"question",
"skill",
"repo_clone",
"repo_overview",
"invalid",
] as const
type ToolKind = (typeof TOOL_KINDS)[number]
// Tool generators take the helpers + return a ToolCall, or null if the tool
// can't be produced right now (e.g. apply_patch needs an existing file).
type ToolGen = (h: Helpers) => ToolCall | null
const TOOL_GENERATORS: Record<ToolKind, ToolGen> = {
apply_patch: (h) => {
const existing = h.fs.all()
if (existing.length === 0) return null
const file = `/opencode/${h.pick(existing)}`
const before = h.pick(["export const", "return", "function", "import"])
const after = h.pick(["export default", "return undefined", "async function", "import type"])
return {
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "apply_patch",
input: {
patchText: `*** Begin Patch\n*** Update File: ${file}\n@@\n- ${before}\n+ ${after}\n*** End Patch\n`,
},
}
},
edit: (h) => {
const existing = h.fs.all()
if (existing.length === 0) return null
return {
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "edit",
input: {
filePath: `/opencode/${h.pick(existing)}`,
oldString: h.pick(["return null", "// TODO", "const x = 1", "if (true)"]),
newString: h.pick(["return undefined", "// fixed", "const x = 2", "if (cond)"]),
...(h.maybe(0.2) ? { replaceAll: true } : {}),
},
}
},
write: (h) => {
const candidates = NEW_FILE_CANDIDATES.filter((p) => !h.fs.has(p))
const file = candidates.length > 0 ? h.pick(candidates) : h.pick(NEW_FILE_CANDIDATES)
const content = h.pick([
"export const value = 42\n",
"// generated\nexport default {}\n",
"TODO: fill in\n",
'{\n "version": "0.0.1"\n}\n',
])
// Track the write so future apply_patch/edit can target it.
h.fs.add(file)
return {
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "write",
input: { filePath: `/opencode/${file}`, content },
}
},
read: (h) => {
const existing = h.fs.all()
if (existing.length === 0) return null
return {
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "read",
input: {
filePath: `/opencode/${h.pick(existing)}`,
...(h.maybe(0.3) ? { offset: h.int(1, 50) } : {}),
...(h.maybe(0.3) ? { limit: h.int(10, 200) } : {}),
},
}
},
grep: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "grep",
input: {
pattern: h.pick(PATTERNS),
...(h.maybe(0.5) ? { path: h.pick(["src", "src/feature", "src/util", "test", "docs"]) } : {}),
...(h.maybe(0.5) ? { include: h.pick(["*.ts", "*.tsx", "*.{ts,tsx}", "*.md"]) } : {}),
},
}),
glob: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "glob",
input: {
pattern: h.pick(GLOBS),
...(h.maybe(0.3) ? { path: h.pick(["src", "test", "docs"]) } : {}),
},
}),
bash: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "bash",
input: {
command: h.pick(COMMANDS),
description: h.pick(["List files", "Show status", "Print working dir", "Run tests"]),
...(h.maybe(0.2) ? { timeout: h.int(1000, 60000) } : {}),
},
}),
todowrite: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "todowrite",
input: {
todos: Array.from({ length: h.int(1, 4) }, () => ({
content: h.pick([
"Investigate failing test",
"Refactor layer composition",
"Add typecheck step",
"Update docs",
"Bump dependencies",
]),
status: h.pick(["pending", "in_progress", "completed", "cancelled"]),
priority: h.pick(["high", "medium", "low"]),
})),
},
}),
webfetch: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "webfetch",
input: {
url: h.pick(WEB_URLS),
...(h.maybe(0.4) ? { format: h.pick(["markdown", "text", "html"]) } : {}),
...(h.maybe(0.2) ? { timeout: h.int(5, 60) } : {}),
},
}),
websearch: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "websearch",
input: {
query: h.pick(SEARCH_QUERIES),
...(h.maybe(0.3) ? { numResults: h.int(1, 10) } : {}),
...(h.maybe(0.3) ? { type: h.pick(["auto", "fast", "deep"]) } : {}),
},
}),
lsp: (h) => {
const existing = h.fs.all().filter((p) => /\.(ts|tsx|js|jsx)$/.test(p))
const file = existing.length > 0 ? `/opencode/${h.pick(existing)}` : "/opencode/src/index.ts"
return {
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "lsp",
input: {
file,
action: h.pick(["definition", "references", "hover", "documentSymbol", "implementation"]),
...(h.maybe(0.5) ? { position: { line: h.int(0, 100), character: h.int(0, 80) } } : {}),
},
}
},
task: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "task",
input: {
description: h.pick(["explore feature", "audit dependency graph", "find usage"]),
prompt: h.pick([
"Look through src/ and summarize the entry points.",
"Find all call sites for `Bus.publish` and explain what they publish.",
]),
subagent_type: h.pick(SUBAGENTS),
},
}),
task_status: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "task_status",
input: { task_id: `task-${h.int(1, 50).toString(36)}` },
}),
plan: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "plan",
input: {
summary: h.pick([
"Refactor the layer graph to remove cycles",
"Add diagnostic logging then propose a fix",
"Extract a helper and add tests",
]),
},
}),
question: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "question",
input: {
questions: [
{
question: h.pick(["Which directory should I scaffold under?", "Approve the rename?"]),
header: h.pick(["Pick a directory", "Approve rename"]),
options: [
{ label: "Yes", description: "Approve" },
{ label: "No", description: "Decline" },
],
},
],
},
}),
skill: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "skill",
input: { name: h.pick(SKILLS) },
}),
repo_clone: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "repo_clone",
input: {
url: h.pick(["https://github.com/anomalyco/opencode", "https://github.com/effect-ts/effect"]),
...(h.maybe(0.4) ? { ref: h.pick(["main", "dev", "v1.0.0"]) } : {}),
},
}),
repo_overview: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "repo_overview",
input: {
...(h.maybe(0.5) ? { path: h.pick(["src", "test", "docs"]) } : {}),
...(h.maybe(0.3) ? { depth: h.int(1, 4) } : {}),
},
}),
invalid: (h) => ({
type: "tool-call",
toolCallId: h.nextToolCallId(),
toolName: "invalid",
input: {
tool: h.pick(["foo", "bar", "definitely_not_a_tool"]),
error: h.pick(["unknown tool", "missing argument"]),
},
}),
}
const DEFAULT_TOOL_WEIGHTS: Record<ToolKind, number> = {
apply_patch: 4,
edit: 3,
write: 3,
read: 4,
grep: 3,
glob: 3,
bash: 2,
todowrite: 2,
webfetch: 2,
websearch: 2,
lsp: 2,
task: 2,
task_status: 1,
plan: 2,
question: 1,
skill: 1,
repo_clone: 1,
repo_overview: 1,
invalid: 1,
}
// ─── Step shapes ─────────────────────────────────────────────────────────────
type StepItem =
| { type: "text"; content: string }
| { type: "thinking"; content: string }
| ToolCall
type ShapeKind =
| "text"
| "multiText"
| "thinkText"
| "textToolCall"
| "thinkTextToolCall"
| "multiToolCall"
| "thinkOnly"
| "longNarrative"
| "soloToolCall"
interface ShapeContext {
h: Helpers
pickTool: () => ToolCall | null
}
const SHAPE_BUILDERS: Record<ShapeKind, (ctx: ShapeContext) => StepItem[] | null> = {
text: ({ h }) => [{ type: "text", content: h.pick(PLAIN_TEXT) }],
multiText: ({ h }) =>
Array.from({ length: h.int(2, 5) }, () => ({ type: "text" as const, content: h.pick(PLAIN_TEXT) })),
thinkText: ({ h }) => [
{ type: "thinking", content: h.pick(THINKING) },
{ type: "text", content: h.pick(PLAIN_TEXT) },
],
textToolCall: ({ h, pickTool }) => {
const tc = pickTool()
if (!tc) return null
return [{ type: "text", content: h.pick(PLAIN_TEXT) }, tc]
},
thinkTextToolCall: ({ h, pickTool }) => {
const tc = pickTool()
if (!tc) return null
return [
{ type: "thinking", content: h.pick(THINKING) },
{ type: "text", content: h.pick(PLAIN_TEXT) },
tc,
]
},
multiToolCall: ({ h, pickTool }) => {
const count = h.int(2, 4)
const tcs: ToolCall[] = []
for (let i = 0; i < count; i++) {
const tc = pickTool()
if (tc) tcs.push(tc)
}
if (tcs.length === 0) return null
return [{ type: "text", content: h.pick(PLAIN_TEXT) }, ...tcs]
},
thinkOnly: ({ h }) => [{ type: "thinking", content: h.pick(THINKING) }],
longNarrative: ({ h }) =>
Array.from({ length: h.int(5, 8) }, () => ({ type: "text" as const, content: h.pick(PLAIN_TEXT) })),
soloToolCall: ({ pickTool }) => {
const tc = pickTool()
if (!tc) return null
return [tc]
},
}
const DEFAULT_SHAPE_WEIGHTS: Record<ShapeKind, number> = {
text: 3,
multiText: 2,
thinkText: 2,
textToolCall: 5,
thinkTextToolCall: 3,
multiToolCall: 2,
thinkOnly: 1,
longNarrative: 2,
soloToolCall: 2,
}
// ─── Weighted picker ─────────────────────────────────────────────────────────
function weightedPicker<K extends string>(weights: Record<K, number>, rand: () => number) {
const entries = Object.entries(weights) as [K, number][]
const positive = entries.filter(([, w]) => w > 0)
if (positive.length === 0) throw new Error("All weights are zero")
const total = positive.reduce((sum, [, w]) => sum + w, 0)
return () => {
let r = rand() * total
for (const [k, w] of positive) {
r -= w
if (r <= 0) return k
}
return positive[positive.length - 1]![0]
}
}
// ─── Main ────────────────────────────────────────────────────────────────────
function main() {
const opts = parseArgs(process.argv.slice(2))
const rand = mulberry32(opts.seed)
const pick = <T,>(items: readonly T[]): T => items[Math.floor(rand() * items.length)]!
const int = (min: number, max: number) => min + Math.floor(rand() * (max - min + 1))
const maybe = (p: number) => rand() < p
let toolCallSeq = 0
const nextToolCallId = () => `tc-${(++toolCallSeq).toString(36)}`
const fs = new FsModel()
if (opts.seedFiles) for (const f of SEED_FILES) fs.seed(f.path, f.content)
const helpers: Helpers = { rand, pick, int, maybe, nextToolCallId, fs }
const enabledTools = (TOOL_KINDS as readonly ToolKind[]).filter(
(t) => !opts.tools || opts.tools.has(t),
)
if (enabledTools.length === 0) throw new Error("--tools filter excluded every tool")
const toolWeights = Object.fromEntries(
enabledTools.map((t) => [t, opts.toolWeights[t] ?? DEFAULT_TOOL_WEIGHTS[t]]),
) as Record<ToolKind, number>
const pickToolKind = weightedPicker(toolWeights, rand)
const enabledShapes = (Object.keys(DEFAULT_SHAPE_WEIGHTS) as ShapeKind[]).filter(
(s) => !opts.shapes || opts.shapes.has(s),
)
if (enabledShapes.length === 0) throw new Error("--shapes filter excluded every shape")
const shapeWeights = Object.fromEntries(
enabledShapes.map((s) => [s, opts.shapeWeights[s] ?? DEFAULT_SHAPE_WEIGHTS[s]]),
) as Record<ShapeKind, number>
const pickShapeKind = weightedPicker(shapeWeights, rand)
// Resolve a tool, retrying up to N times if a generator returns null (e.g.
// apply_patch with no existing files). If every retry fails, fall back to
// `text` step shape via returning null upstream.
const tryPickTool = (): ToolCall | null => {
for (let i = 0; i < 6; i++) {
const tc = TOOL_GENERATORS[pickToolKind() as ToolKind](helpers)
if (tc) return tc
}
return null
}
type LLMScript = {
steps: StepItem[][]
usage?: { inputTokens: number; outputTokens: number; totalTokens: number }
finish?: "stop" | "tool-calls" | "length" | "unknown"
}
function makeScript(): LLMScript {
// Try up to 4 different shapes; if none yields a step (e.g. all
// tool-shape variants returned null), fall back to a `text` step.
let step: StepItem[] | null = null
for (let i = 0; i < 4 && !step; i++) {
const shape = pickShapeKind() as ShapeKind
step = SHAPE_BUILDERS[shape]({ h: helpers, pickTool: tryPickTool })
}
if (!step) step = [{ type: "text", content: pick(PLAIN_TEXT) }]
const hasToolCall = step.some((item) => item.type === "tool-call")
const finish: LLMScript["finish"] = hasToolCall ? "tool-calls" : pick(FINISH_REASONS)
const inputTokens = int(20, 600)
const outputTokens = int(4, 250)
return {
steps: [step],
usage: { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens },
finish,
}
}
// Title-gen padding: each user turn the small-model gets invoked too. To
// keep the user-visible follow-up text from falling back to the default
// "Simulation mock response.", pad the queue by `turns` extra short scripts.
const titlePadding = opts.enableTitles ? opts.turns : 0
const scripts: LLMScript[] = Array.from({ length: opts.total + titlePadding }, makeScript)
// ─── User actions ──────────────────────────────────────────────────────────
const userActions = Array.from({ length: opts.turns }, (_, i) => [
{ type: "typeText", text: PROMPTS[i % PROMPTS.length]! },
{ type: "pressEnter" },
{ type: "wait", ms: 60 },
]).flat()
// Seed-file writes go FIRST so the FS exists before user turns run.
const seedWrites = fs.seededWrites().map((f) => ({
type: "writeFile",
path: f.path,
content: f.content,
}))
const script = {
_comment: `Generated by generate.ts. Seed: 0x${opts.seed.toString(16)}. Total LLM scripts: ${opts.total} (+${titlePadding} title padding = ${scripts.length}). Turns: ${opts.turns}. Tools enabled: ${enabledTools.length}. Pre-seeded files: ${fs.seededWrites().length}.`,
actions: [
{ type: "pressKey", key: "x", modifiers: { ctrl: true } },
{ type: "pressKey", key: "b" },
...seedWrites,
{ type: "enqueueLLM", scripts },
...userActions,
],
}
const outPath = path.resolve(opts.out)
writeFileSync(outPath, JSON.stringify(script, null, 2) + "\n")
// ─── Summary ───────────────────────────────────────────────────────────────
const byFinish: Record<string, number> = {}
for (const s of scripts) {
const k = s.finish ?? "stop"
byFinish[k] = (byFinish[k] ?? 0) + 1
}
const byTool: Record<string, number> = {}
let totalToolCalls = 0
for (const s of scripts) {
for (const step of s.steps) {
for (const item of step) {
if (item.type === "tool-call") {
byTool[item.toolName] = (byTool[item.toolName] ?? 0) + 1
totalToolCalls++
}
}
}
}
console.log(`Wrote ${outPath}`)
console.log(`Total scripts: ${scripts.length} (${opts.total} primary + ${titlePadding} title padding)`)
console.log(`User turns: ${opts.turns}`)
console.log(`Seeded files: ${fs.seededWrites().length}`)
console.log(`Finish reasons:`, byFinish)
console.log(`Tool calls (${totalToolCalls} total):`, byTool)
}
main()

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,306 @@
#!/usr/bin/env bun
/**
* Simulation script runner.
*
* Drives a simulation script against a running `bun dev simulate` process via
* the simulation MCP HTTP endpoint. Loads the script, steps through it in
* configurable chunks, and checks the in-memory log buffer after each chunk.
* Stops on the first error (configurable level) and prints the failing entries
* plus the most recent step that was executed.
*
* Usage:
* bun test/testing/simulation/scripts/run.ts <script.json> [options]
*
* Options:
* --mcp <url> MCP endpoint. Default: http://127.0.0.1:43110/mcp
* --chunk <n> Actions per step batch. Default: 3
* --max-steps <n> Hard cap on step calls. Default: unlimited.
* --level <lvl> Stop on entries at or above this level.
* One of DEBUG | INFO | WARN | ERROR. Default: ERROR.
* --message-includes <s> Only stop when matching message substring.
* --service-includes <s> Only stop when matching tag.service substring.
* --reset Reset simulation state + restart TUI before loading.
* --no-reset Skip the reset/restart (default).
* --keep-going Don't stop on errors; print them and continue.
* --quiet Suppress per-batch progress output.
* --json Emit a single JSON summary at the end.
* --check-every <n> Check logs only every N batches. Default: 1.
*
* Example:
* bun test/testing/simulation/scripts/run.ts patches.json \
* --reset --chunk 3 --level ERROR
*/
interface Options {
scriptPath: string
mcpUrl: string
chunk: number
maxSteps: number
level: "DEBUG" | "INFO" | "WARN" | "ERROR"
messageIncludes?: string
serviceIncludes?: string
reset: boolean
keepGoing: boolean
quiet: boolean
json: boolean
checkEvery: number
}
function parseArgs(argv: string[]): Options {
if (argv.length === 0 || argv[0] === "--help" || argv[0] === "-h") {
printHelp()
process.exit(0)
}
const out: Options = {
scriptPath: argv[0]!,
mcpUrl: "http://127.0.0.1:43110/mcp",
chunk: 3,
maxSteps: -1,
level: "ERROR",
reset: false,
keepGoing: false,
quiet: false,
json: false,
checkEvery: 1,
}
for (let i = 1; i < argv.length; i++) {
const a = argv[i]!
const next = () => {
const v = argv[++i]
if (v === undefined) throw new Error(`${a} requires a value`)
return v
}
switch (a) {
case "--mcp":
out.mcpUrl = next()
break
case "--chunk":
out.chunk = Number(next())
break
case "--max-steps":
out.maxSteps = Number(next())
break
case "--level": {
const v = next().toUpperCase()
if (!["DEBUG", "INFO", "WARN", "ERROR"].includes(v)) {
throw new Error(`--level must be DEBUG|INFO|WARN|ERROR, got ${v}`)
}
out.level = v as Options["level"]
break
}
case "--message-includes":
out.messageIncludes = next()
break
case "--service-includes":
out.serviceIncludes = next()
break
case "--reset":
out.reset = true
break
case "--no-reset":
out.reset = false
break
case "--keep-going":
out.keepGoing = true
break
case "--quiet":
out.quiet = true
break
case "--json":
out.json = true
break
case "--check-every":
out.checkEvery = Number(next())
break
case "--help":
case "-h":
printHelp()
process.exit(0)
default:
throw new Error(`Unknown argument: ${a}`)
}
}
return out
}
function printHelp() {
console.log(`Simulation script runner.
Usage:
bun run.ts <script.json> [options]
Options:
--mcp <url> MCP endpoint (default http://127.0.0.1:43110/mcp)
--chunk <n> Actions per step batch (default 3)
--max-steps <n> Hard cap on step calls (default unlimited)
--level <lvl> Stop level: DEBUG|INFO|WARN|ERROR (default ERROR)
--message-includes <s> Only stop when message includes substring
--service-includes <s> Only stop when tag.service includes substring
--reset Reset sim state + restart TUI before load
--no-reset Skip reset (default)
--keep-going Don't stop on errors; continue to end
--quiet Suppress per-batch progress
--json Emit JSON summary at the end
--check-every <n> Check logs every N batches (default 1)
`)
}
// ─── MCP client ──────────────────────────────────────────────────────────────
let rpcId = 0
async function mcpCall(mcpUrl: string, name: string, args: Record<string, unknown> = {}) {
const response = await fetch(mcpUrl, {
method: "POST",
headers: {
"content-type": "application/json",
accept: "application/json, text/event-stream",
},
body: JSON.stringify({
jsonrpc: "2.0",
id: ++rpcId,
method: "tools/call",
params: { name, arguments: args },
}),
})
if (!response.ok) throw new Error(`${name}: HTTP ${response.status}`)
const text = await response.text()
// The MCP server may respond with a single JSON line or with SSE-style
// chunks. Find the first `{"result"` line either way.
const jsonLine = text
.split("\n")
.map((line) => line.replace(/^data:\s*/, ""))
.find((line) => line.trim().startsWith('{"result"') || line.trim().startsWith('{"error"'))
if (!jsonLine) throw new Error(`${name}: no JSON-RPC response in:\n${text.slice(0, 500)}`)
const envelope = JSON.parse(jsonLine)
if (envelope.error) {
throw new Error(`${name}: ${envelope.error.message ?? JSON.stringify(envelope.error)}`)
}
const content = envelope.result?.content?.[0]?.text
if (typeof content !== "string") {
throw new Error(`${name}: unexpected MCP envelope:\n${JSON.stringify(envelope).slice(0, 500)}`)
}
// Tool result `text` payloads are JSON-encoded.
return JSON.parse(content)
}
// ─── Logic ───────────────────────────────────────────────────────────────────
interface LogEntry {
time: string
level: "DEBUG" | "INFO" | "WARN" | "ERROR"
tags: Record<string, unknown>
message: string
}
async function run(opts: Options) {
const log = (msg: string) => {
if (!opts.quiet) console.log(msg)
}
if (opts.reset) {
log("Resetting simulation state...")
await mcpCall(opts.mcpUrl, "simulation_control_reset")
log("Restarting TUI...")
await mcpCall(opts.mcpUrl, "simulation_restart")
}
log("Clearing log buffer...")
await mcpCall(opts.mcpUrl, "simulation_log_clear")
log(`Loading script: ${opts.scriptPath}`)
const loaded = await mcpCall(opts.mcpUrl, "simulation_script_load", {
path: opts.scriptPath,
replace: true,
})
log(` id=${loaded.id} total=${loaded.total}`)
const total = loaded.total as number
let batches = 0
let cursor = 0
let stopReason: "completed" | "max-steps" | "error" = "completed"
let stoppingEntries: LogEntry[] | undefined
let stoppingExecuted: unknown[] | undefined
while (cursor < total) {
if (opts.maxSteps > 0 && batches >= opts.maxSteps) {
stopReason = "max-steps"
break
}
const step = await mcpCall(opts.mcpUrl, "simulation_script_step", { steps: opts.chunk })
cursor = step.state.cursor as number
batches++
if (!opts.quiet) {
const lastKinds = (step.executed as { type: string }[]).map((a) => a.type).join(",")
log(`batch ${batches}: cursor ${cursor}/${total}${lastKinds}`)
}
if (batches % opts.checkEvery !== 0 && cursor < total) continue
const logResp = await mcpCall(opts.mcpUrl, "simulation_log_get", {
level: opts.level,
...(opts.messageIncludes ? { messageIncludes: opts.messageIncludes } : {}),
...(opts.serviceIncludes ? { serviceIncludes: opts.serviceIncludes } : {}),
})
const entries = (logResp.entries ?? []) as LogEntry[]
if (entries.length > 0) {
if (opts.keepGoing) {
if (!opts.quiet) {
console.log(
`${entries.length} ${opts.level} entries — continuing (--keep-going)`,
)
}
// Clear so we only see new ones in subsequent batches.
await mcpCall(opts.mcpUrl, "simulation_log_clear")
} else {
stopReason = "error"
stoppingEntries = entries
stoppingExecuted = step.executed
break
}
}
}
const status = await mcpCall(opts.mcpUrl, "simulation_script_status")
if (opts.json) {
process.stdout.write(
JSON.stringify(
{
stopReason,
batches,
cursor,
total,
status: status.status,
stoppingEntries,
stoppingExecuted,
},
null,
2,
) + "\n",
)
return
}
console.log()
console.log(`Stopped: ${stopReason}`)
console.log(`Batches: ${batches}`)
console.log(`Cursor: ${cursor}/${total}`)
if (stopReason === "error" && stoppingEntries) {
console.log()
console.log(`${stoppingEntries.length} ${opts.level} entries — failing batch executed:`)
console.log(JSON.stringify(stoppingExecuted, null, 2))
console.log()
console.log(`${opts.level} entries:`)
console.log(JSON.stringify(stoppingEntries, null, 2))
process.exitCode = 1
}
}
try {
await run(parseArgs(process.argv.slice(2)))
} catch (err) {
console.error(err instanceof Error ? err.message : err)
process.exit(1)
}