wip
This commit is contained in:
parent
53d45b2894
commit
241b7b5792
13 changed files with 92434 additions and 26 deletions
19
packages/opencode/gen.json
Normal file
19
packages/opencode/gen.json
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
Simulation script runner.
|
||||
|
||||
Usage:
|
||||
bun run.ts <script.json> [options]
|
||||
|
||||
Options:
|
||||
--mcp <url> MCP endpoint (default http://127.0.0.1:43110/mcp)
|
||||
--chunk <n> Actions per step batch (default 3)
|
||||
--max-steps <n> Hard cap on step calls (default unlimited)
|
||||
--level <lvl> Stop level: DEBUG|INFO|WARN|ERROR (default ERROR)
|
||||
--message-includes <s> Only stop when message includes substring
|
||||
--service-includes <s> Only stop when tag.service includes substring
|
||||
--reset Reset sim state + restart TUI before load
|
||||
--no-reset Skip reset (default)
|
||||
--keep-going Don't stop on errors; continue to end
|
||||
--quiet Suppress per-batch progress
|
||||
--json Emit JSON summary at the end
|
||||
--check-every <n> Check logs every N batches (default 1)
|
||||
|
||||
45316
packages/opencode/generated.json
Normal file
45316
packages/opencode/generated.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -373,38 +373,133 @@ async function discoverInstances(input: { startPort?: number; maxPorts?: number;
|
|||
return { instances: instances(), discovered: found, scanned: { startPort, maxPorts, consecutiveFailures } }
|
||||
}
|
||||
|
||||
type ScriptAction = z.infer<typeof ScriptActionSchema>
|
||||
type ScriptCounts = { uiActions: number; fileWrites: number; llmScriptsQueued: number; waits: number }
|
||||
|
||||
async function executeAction(options: Options, action: ScriptAction, counts: ScriptCounts) {
|
||||
if (action.type === "writeFile") {
|
||||
await control(options, "POST", "/experimental/simulation/filesystem/write", {
|
||||
path: action.path,
|
||||
content: action.content,
|
||||
})
|
||||
counts.fileWrites++
|
||||
return
|
||||
}
|
||||
if (action.type === "enqueueLLM") {
|
||||
await control(options, "POST", "/experimental/simulation/llm/enqueue", { scripts: action.scripts })
|
||||
counts.llmScriptsQueued += action.scripts.length
|
||||
return
|
||||
}
|
||||
if (action.type === "wait") {
|
||||
await new Promise((resolve) => setTimeout(resolve, action.ms ?? 1_000))
|
||||
await current(options).harness.renderOnce()
|
||||
counts.waits++
|
||||
return
|
||||
}
|
||||
await SimulationActions.execute(current(options).harness, action)
|
||||
counts.uiActions++
|
||||
}
|
||||
|
||||
async function runScript(options: Options, file: string) {
|
||||
const parsed = ScriptSchema.parse(await Bun.file(file).json())
|
||||
const actions = Array.isArray(parsed) ? parsed : parsed.actions
|
||||
const counts = { uiActions: 0, fileWrites: 0, llmScriptsQueued: 0, waits: 0 }
|
||||
|
||||
for (const action of actions) {
|
||||
if (action.type === "writeFile") {
|
||||
await control(options, "POST", "/experimental/simulation/filesystem/write", {
|
||||
path: action.path,
|
||||
content: action.content,
|
||||
})
|
||||
counts.fileWrites++
|
||||
continue
|
||||
}
|
||||
if (action.type === "enqueueLLM") {
|
||||
await control(options, "POST", "/experimental/simulation/llm/enqueue", { scripts: action.scripts })
|
||||
counts.llmScriptsQueued += action.scripts.length
|
||||
continue
|
||||
}
|
||||
if (action.type === "wait") {
|
||||
await new Promise((resolve) => setTimeout(resolve, action.ms ?? 1_000))
|
||||
await current(options).harness.renderOnce()
|
||||
counts.waits++
|
||||
continue
|
||||
}
|
||||
await SimulationActions.execute(current(options).harness, action)
|
||||
counts.uiActions++
|
||||
}
|
||||
|
||||
const counts: ScriptCounts = { uiActions: 0, fileWrites: 0, llmScriptsQueued: 0, waits: 0 }
|
||||
for (const action of actions) await executeAction(options, action, counts)
|
||||
return { file, actions: actions.length, ...counts, snapshot: snapshot(options) }
|
||||
}
|
||||
|
||||
// ─── Step-controlled script execution ───────────────────────────────────────
|
||||
//
|
||||
// `simulation_script_load` parses a script and stores its actions in process
|
||||
// memory keyed by a generated id. The script is NOT executed yet. Subsequent
|
||||
// calls to `simulation_script_step` advance the cursor by one or more actions
|
||||
// at a time, returning the snapshot and updated counts after each batch. This
|
||||
// lets MCP clients drive scripts at their own pace and inspect state between
|
||||
// steps. Only one script is active at a time per process; loading a new one
|
||||
// while a previous one is still pending requires either consuming it to
|
||||
// completion, calling `simulation_script_cancel`, or specifying replace=true.
|
||||
|
||||
interface LoadedScript {
|
||||
readonly id: string
|
||||
readonly file: string | null
|
||||
readonly actions: ScriptAction[]
|
||||
cursor: number
|
||||
readonly counts: ScriptCounts
|
||||
readonly loadedAt: string
|
||||
}
|
||||
|
||||
let loaded: LoadedScript | undefined
|
||||
let loadSeq = 0
|
||||
|
||||
function loadedSummary(state: LoadedScript) {
|
||||
return {
|
||||
id: state.id,
|
||||
file: state.file,
|
||||
total: state.actions.length,
|
||||
cursor: state.cursor,
|
||||
remaining: state.actions.length - state.cursor,
|
||||
done: state.cursor >= state.actions.length,
|
||||
counts: { ...state.counts },
|
||||
loadedAt: state.loadedAt,
|
||||
}
|
||||
}
|
||||
|
||||
async function loadScript(input: {
|
||||
file?: string
|
||||
script?: unknown
|
||||
replace?: boolean
|
||||
}) {
|
||||
if (loaded && loaded.cursor < loaded.actions.length && !input.replace) {
|
||||
throw new Error(
|
||||
`A script is already loaded (id=${loaded.id}, ${loaded.actions.length - loaded.cursor} actions remaining). Pass replace=true or call simulation_script_cancel first.`,
|
||||
)
|
||||
}
|
||||
const raw = input.file
|
||||
? await Bun.file(input.file).json()
|
||||
: (input.script ?? (() => {
|
||||
throw new Error("simulation_script_load requires either `file` or `script`.")
|
||||
})())
|
||||
const parsed = ScriptSchema.parse(raw)
|
||||
const actions = Array.isArray(parsed) ? parsed : parsed.actions
|
||||
loaded = {
|
||||
id: `script-${(++loadSeq).toString(36)}`,
|
||||
file: input.file ?? null,
|
||||
actions: [...actions] as ScriptAction[],
|
||||
cursor: 0,
|
||||
counts: { uiActions: 0, fileWrites: 0, llmScriptsQueued: 0, waits: 0 },
|
||||
loadedAt: new Date().toISOString(),
|
||||
}
|
||||
return loadedSummary(loaded)
|
||||
}
|
||||
|
||||
async function stepScript(options: Options, input: { steps?: number; renderEach?: boolean }) {
|
||||
if (!loaded) throw new Error("No script loaded. Call simulation_script_load first.")
|
||||
const max = input.steps ?? 1
|
||||
const executed: ScriptAction[] = []
|
||||
for (let i = 0; i < max && loaded.cursor < loaded.actions.length; i++) {
|
||||
const action = loaded.actions[loaded.cursor]!
|
||||
executed.push(action)
|
||||
await executeAction(options, action, loaded.counts)
|
||||
loaded.cursor++
|
||||
if (input.renderEach && i < max - 1) await current(options).harness.renderOnce()
|
||||
}
|
||||
return {
|
||||
executed,
|
||||
state: loadedSummary(loaded),
|
||||
snapshot: snapshot(options),
|
||||
}
|
||||
}
|
||||
|
||||
function cancelScript() {
|
||||
const was = loaded ? loadedSummary(loaded) : null
|
||||
loaded = undefined
|
||||
return { cancelled: was !== null, was }
|
||||
}
|
||||
|
||||
function statusScript() {
|
||||
return loaded ? loadedSummary(loaded) : null
|
||||
}
|
||||
|
||||
async function runOnTargets<A>(
|
||||
options: Options,
|
||||
target: z.infer<typeof TargetSchema>,
|
||||
|
|
@ -504,6 +599,48 @@ function createServer(options: Options) {
|
|||
async (input) => toolResult(await runScript(options, input.path)),
|
||||
)
|
||||
|
||||
server.registerTool(
|
||||
"simulation_script_load",
|
||||
{
|
||||
description:
|
||||
"Load a simulation script into memory WITHOUT executing it. Pass `path` to load from a JSON file on disk, or `script` to load inline JSON. Returns the parsed action count and a script id. Use `simulation_script_step` to execute actions one (or N) at a time. Only one script may be loaded at once unless `replace` is true.",
|
||||
inputSchema: z.object({
|
||||
path: z.string().optional(),
|
||||
script: z.any().optional(),
|
||||
replace: z.boolean().optional(),
|
||||
}),
|
||||
},
|
||||
async (input) => toolResult(await loadScript({ file: input.path, script: input.script, replace: input.replace })),
|
||||
)
|
||||
|
||||
server.registerTool(
|
||||
"simulation_script_step",
|
||||
{
|
||||
description:
|
||||
"Execute the next action(s) of the loaded script and return the snapshot afterwards. Defaults to one step. Pass `steps` to advance multiple actions in a single call (1-100). When `renderEach` is true, the simulated TUI is forced to render between steps.",
|
||||
inputSchema: z.object({
|
||||
steps: z.number().int().min(1).max(100).optional(),
|
||||
renderEach: z.boolean().optional(),
|
||||
}),
|
||||
},
|
||||
async (input) => toolResult(await stepScript(options, { steps: input.steps, renderEach: input.renderEach })),
|
||||
)
|
||||
|
||||
server.registerTool(
|
||||
"simulation_script_status",
|
||||
{
|
||||
description:
|
||||
"Return the currently-loaded script's progress: id, total actions, cursor, remaining, and execution counts. Returns null when nothing is loaded.",
|
||||
},
|
||||
async () => toolResult({ status: statusScript() }),
|
||||
)
|
||||
|
||||
server.registerTool(
|
||||
"simulation_script_cancel",
|
||||
{ description: "Discard the currently-loaded script (if any). Subsequent step calls fail until a new script is loaded." },
|
||||
async () => toolResult(cancelScript()),
|
||||
)
|
||||
|
||||
if ("runtime" in options) {
|
||||
server.registerTool("simulation_restart", { description: "Restart the simulated TUI and backend while keeping MCP alive." }, async () =>
|
||||
toolResult(await options.runtime.restart()),
|
||||
|
|
@ -582,6 +719,45 @@ function createServer(options: Options) {
|
|||
},
|
||||
)
|
||||
|
||||
server.registerTool(
|
||||
"simulation_log_get",
|
||||
{
|
||||
description:
|
||||
"Return the in-memory log buffer captured by `@opencode-ai/core/util/log` inside the simulated backend (capped at 5000 most recent entries). Filter by `level` to return only entries at or above that level. Also supports optional `limit` and substring filters.",
|
||||
inputSchema: z.object({
|
||||
level: z.enum(["DEBUG", "INFO", "WARN", "ERROR"]).optional(),
|
||||
limit: z.number().int().min(1).max(5000).optional(),
|
||||
messageIncludes: z.string().optional(),
|
||||
serviceIncludes: z.string().optional(),
|
||||
}),
|
||||
},
|
||||
async (input) => {
|
||||
const data = await control(options, "GET", "/experimental/simulation/log/entries")
|
||||
let entries = (data?.entries ?? []) as Array<{
|
||||
time: string
|
||||
level: "DEBUG" | "INFO" | "WARN" | "ERROR"
|
||||
tags: Record<string, unknown>
|
||||
message: string
|
||||
}>
|
||||
if (input.level) {
|
||||
const priority = { DEBUG: 0, INFO: 1, WARN: 2, ERROR: 3 } as const
|
||||
const threshold = priority[input.level]
|
||||
entries = entries.filter((e) => priority[e.level] >= threshold)
|
||||
}
|
||||
if (input.messageIncludes) entries = entries.filter((e) => e.message.includes(input.messageIncludes!))
|
||||
if (input.serviceIncludes)
|
||||
entries = entries.filter((e) => String(e.tags?.service ?? "").includes(input.serviceIncludes!))
|
||||
if (typeof input.limit === "number") entries = entries.slice(-input.limit)
|
||||
return toolResult({ entries, total: entries.length })
|
||||
},
|
||||
)
|
||||
|
||||
server.registerTool(
|
||||
"simulation_log_clear",
|
||||
{ description: "Clear the simulated backend's in-memory log buffer." },
|
||||
async () => toolResult(await control(options, "POST", "/experimental/simulation/log/clear")),
|
||||
)
|
||||
|
||||
if (masterEnabled()) {
|
||||
server.registerTool(
|
||||
"simulation_instances_discover",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ if (process.env.OPENCODE_SIMULATION_CWD) {
|
|||
Global.Path.state = `${process.env.OPENCODE_SIMULATION_CWD}/.local/state/opencode`
|
||||
Global.Path.tmp = `${process.env.OPENCODE_SIMULATION_CWD}/tmp/opencode`
|
||||
Global.Path.bin = `${Global.Path.cache}/bin`
|
||||
Global.Path.log = `${Global.Path.data}/log`
|
||||
Global.Path.repos = `${Global.Path.data}/repos`
|
||||
Object.defineProperty(process, "cwd", {
|
||||
value: () => process.env.OPENCODE_SIMULATION_CWD!,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { Simulation } from "@/testing/simulation/service"
|
||||
import { Effect } from "effect"
|
||||
import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"
|
||||
import * as Log from "@opencode-ai/core/util/log"
|
||||
|
||||
const ok = { ok: true }
|
||||
|
||||
|
|
@ -56,6 +57,19 @@ export const simulationRoute = HttpRouter.use((router) =>
|
|||
)
|
||||
|
||||
yield* router.add("GET", "/experimental/simulation/snapshot", () => json(simulation.snapshot()))
|
||||
|
||||
yield* router.add("GET", "/experimental/simulation/log/entries", () =>
|
||||
Effect.succeed(HttpServerResponse.jsonUnsafe({ entries: Log.entries() })),
|
||||
)
|
||||
|
||||
yield* router.add("POST", "/experimental/simulation/log/clear", () =>
|
||||
Effect.succeed(
|
||||
HttpServerResponse.jsonUnsafe(((): { cleared: true } => {
|
||||
Log.clearEntries()
|
||||
return { cleared: true }
|
||||
})()),
|
||||
),
|
||||
)
|
||||
}),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -161,9 +161,21 @@ export const SimulatedTypescript: LSPServer.Info = {
|
|||
// backend reports clients for the same files. The simulated root is just
|
||||
// the instance directory — we don't probe the filesystem here because the
|
||||
// simulated FS is in-memory and the only "project" is `/opencode`.
|
||||
extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"],
|
||||
extensions: [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts", ".json", ".md"],
|
||||
root: async (_file, ctx) => ctx.directory,
|
||||
async spawn(_root, _ctx) {
|
||||
// Real LSP server spawn is an actual `child_process.spawn` that crosses an
|
||||
// I/O boundary and forces the surrounding `await` to truly yield to the
|
||||
// microtask queue (and beyond). The simulated path is otherwise fully
|
||||
// synchronous, which lets the calling Effect fiber's context survive
|
||||
// `await server.spawn(...)` in cases where production would lose it.
|
||||
//
|
||||
// To make the simulation a faithful behavioral surrogate of the real
|
||||
// server (and surface bugs like #27880), force a real async yield here
|
||||
// via setTimeout so the awaiting fiber is fully torn down before we
|
||||
// resume.
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 500))
|
||||
|
||||
const { process, serverInput, serverOutput } = createFakeProcess()
|
||||
|
||||
const connection = createMessageConnection(
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
{ "type": "pressKey", "key": "b" },
|
||||
{ "type": "writeFile", "path": "src/greeting.ts", "content": "export function greet(name: string) {\n return `Hi, ${name}`\n}\n" },
|
||||
{
|
||||
"_comment": "Three scripts enqueued: (1) main session prompt — long explanation + apply_patch tool call; (2) post-tool-result follow-up text; (3) title-generation small-model call. Title gen fires in parallel with the main call, so script order is by FIFO race winner. With three scripts queued the post-tool follow-up always finds a real script instead of falling back to the default 'Simulation mock response.'",
|
||||
"type": "enqueueLLM",
|
||||
"scripts": [
|
||||
{
|
||||
|
|
@ -35,6 +36,15 @@
|
|||
],
|
||||
"usage": { "inputTokens": 140, "outputTokens": 16, "totalTokens": 156 },
|
||||
"finish": "stop"
|
||||
},
|
||||
{
|
||||
"steps": [
|
||||
[
|
||||
{ "type": "text", "content": "Friendlier greeting" }
|
||||
]
|
||||
],
|
||||
"usage": { "inputTokens": 20, "outputTokens": 4, "totalTokens": 24 },
|
||||
"finish": "stop"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,578 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Generator for `09_diverse_llm_responses.json`.
|
||||
*
|
||||
* Produces a single simulation script that queues 1000 LLM responses with
|
||||
* maximum diversity across:
|
||||
* - finish reasons (stop, tool-calls, error, length, unknown)
|
||||
* - step shapes (single text, multi-text, thinking + text, text + tool-call,
|
||||
* multiple tool-calls, thinking-only, error-only)
|
||||
* - tool variety: edit, write, apply_patch, read, grep, glob, bash, todowrite,
|
||||
* webfetch, websearch, lsp, task, task_status, plan, question, skill,
|
||||
* repo_clone, repo_overview, invalid
|
||||
* - parameter shapes per tool (varied filePaths, patterns, commands, queries)
|
||||
*
|
||||
* The generator is deterministic (seeded RNG) so the same JSON is regenerated.
|
||||
*
|
||||
* Run with:
|
||||
* bun test/testing/simulation/scripts/09_diverse_llm_responses.gen.ts
|
||||
*
|
||||
* Output: `09_diverse_llm_responses.json` (next to this file).
|
||||
*/
|
||||
|
||||
import { writeFileSync } from "fs"
|
||||
import path from "path"
|
||||
|
||||
const TOTAL = 1000
|
||||
const SEED = 0x09abcdef
|
||||
const OUTPUT_PATH = path.join(import.meta.dirname, "09_diverse_llm_responses.json")
|
||||
|
||||
// ─── Seeded RNG (mulberry32) ─────────────────────────────────────────────────
|
||||
function mulberry32(seed: number) {
|
||||
let state = seed >>> 0
|
||||
return () => {
|
||||
state = (state + 0x6d2b79f5) >>> 0
|
||||
let t = state
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1)
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
||||
}
|
||||
}
|
||||
|
||||
const rand = mulberry32(SEED)
|
||||
const pick = <T>(items: readonly T[]): T => items[Math.floor(rand() * items.length)]!
|
||||
const int = (min: number, max: number) => min + Math.floor(rand() * (max - min + 1))
|
||||
const maybe = (p: number) => rand() < p
|
||||
|
||||
// ─── Vocab ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const FILE_NAMES = [
|
||||
"src/index.ts",
|
||||
"src/server.ts",
|
||||
"src/util/log.ts",
|
||||
"src/util/string.ts",
|
||||
"src/feature/auth.ts",
|
||||
"src/feature/cart.ts",
|
||||
"src/feature/checkout.tsx",
|
||||
"src/components/Button.tsx",
|
||||
"src/components/Modal.tsx",
|
||||
"src/lib/db.ts",
|
||||
"src/lib/cache.ts",
|
||||
"src/api/users.ts",
|
||||
"src/api/orders.ts",
|
||||
"test/index.test.ts",
|
||||
"test/auth.test.ts",
|
||||
"test/cart.test.ts",
|
||||
"scripts/build.ts",
|
||||
"scripts/deploy.sh",
|
||||
"config/eslint.json",
|
||||
"config/tsconfig.json",
|
||||
"package.json",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"docs/api.md",
|
||||
"docs/getting-started.md",
|
||||
"Dockerfile",
|
||||
".github/workflows/ci.yml",
|
||||
]
|
||||
|
||||
const DIR_NAMES = ["src", "src/feature", "src/util", "test", "scripts", "config", "docs"]
|
||||
|
||||
const PATTERNS = [
|
||||
"TODO",
|
||||
"FIXME",
|
||||
"console\\.log",
|
||||
"function\\s+\\w+",
|
||||
"import .* from",
|
||||
"export const",
|
||||
"throw new Error",
|
||||
"async\\s+function",
|
||||
"class\\s+\\w+",
|
||||
"interface\\s+\\w+",
|
||||
]
|
||||
|
||||
const GLOBS = [
|
||||
"**/*.ts",
|
||||
"**/*.tsx",
|
||||
"src/**/*.ts",
|
||||
"test/**/*.test.ts",
|
||||
"**/*.{ts,tsx}",
|
||||
"**/*.md",
|
||||
"**/*.json",
|
||||
"scripts/*.sh",
|
||||
]
|
||||
|
||||
const COMMANDS = [
|
||||
"ls -la",
|
||||
"pwd",
|
||||
"cat README.md",
|
||||
"wc -l src/index.ts",
|
||||
"git status",
|
||||
"git log --oneline -5",
|
||||
"git diff --stat",
|
||||
"bun install",
|
||||
"bun test",
|
||||
"bun run build",
|
||||
"npm run lint",
|
||||
"rg TODO",
|
||||
"find . -name '*.ts' -newer package.json",
|
||||
"echo 'hello'",
|
||||
"date",
|
||||
]
|
||||
|
||||
const WEB_URLS = [
|
||||
"https://example.com/api/data",
|
||||
"https://docs.opencode.ai/configuration",
|
||||
"https://github.com/anomalyco/opencode",
|
||||
"https://api.openai.com/v1/models",
|
||||
"https://registry.npmjs.org/effect",
|
||||
"https://nodejs.org/api/fs.html",
|
||||
"https://typescript.org/docs/handbook",
|
||||
]
|
||||
|
||||
const SEARCH_QUERIES = [
|
||||
"rust async error handling",
|
||||
"effect-ts schema validation",
|
||||
"react server components",
|
||||
"typescript discriminated unions",
|
||||
"bun sqlite performance",
|
||||
"lsp protocol initialize",
|
||||
"git rebase squash workflow",
|
||||
]
|
||||
|
||||
const SKILLS = ["customize-opencode", "effect", "improve-codebase-architecture", "gmail"]
|
||||
|
||||
const SUBAGENTS = ["explore", "general"]
|
||||
|
||||
const PLAIN_TEXT = [
|
||||
"Looking at the code now.",
|
||||
"Let me inspect the relevant files.",
|
||||
"I'll start by reading the entrypoint.",
|
||||
"Checking the test suite for related coverage.",
|
||||
"Tracing the call chain through the layer composition.",
|
||||
"This looks like a missing dependency in the layer graph.",
|
||||
"I'll add a small helper to factor out the duplication.",
|
||||
"Renaming the symbol everywhere it's used.",
|
||||
"Bumping the version in package.json.",
|
||||
"Adding a changelog entry.",
|
||||
"Running the formatter.",
|
||||
"Re-running the typecheck.",
|
||||
"All clean — no type errors.",
|
||||
"Tests pass locally.",
|
||||
"Drafting the PR description.",
|
||||
]
|
||||
|
||||
const THINKING = [
|
||||
"Need to figure out which layer is missing the dependency.",
|
||||
"The error stack points at instance-state.ts — likely missing InstanceRef.",
|
||||
"Best to check if the cache is being invalidated correctly.",
|
||||
"Tradeoff: inline vs extract helper. Inline is shorter.",
|
||||
"Race condition seems likely given the await boundary.",
|
||||
"Looking at the diff to spot the regression.",
|
||||
"Need to make sure the test asserts the post-condition.",
|
||||
"Let me re-read the spec to be sure.",
|
||||
]
|
||||
|
||||
const FINISH_REASONS: ("stop" | "tool-calls" | "length" | "unknown")[] = [
|
||||
"stop",
|
||||
"tool-calls",
|
||||
"stop",
|
||||
"tool-calls",
|
||||
"stop",
|
||||
"length",
|
||||
"stop",
|
||||
"unknown",
|
||||
]
|
||||
|
||||
// ─── Tool-call generators ────────────────────────────────────────────────────
|
||||
|
||||
type ToolCall = {
|
||||
type: "tool-call"
|
||||
toolCallId: string
|
||||
toolName: string
|
||||
input: Record<string, unknown>
|
||||
}
|
||||
|
||||
let toolCallSeq = 0
|
||||
const nextToolCallId = () => `tc-${(++toolCallSeq).toString(36)}`
|
||||
|
||||
const TOOL_GENERATORS: ReadonlyArray<() => ToolCall> = [
|
||||
// edit
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "edit",
|
||||
input: {
|
||||
filePath: `/opencode/${pick(FILE_NAMES)}`,
|
||||
oldString: pick(["return null", "// TODO", "const x = 1", "throw new Error(\"!\")", "if (true)"]),
|
||||
newString: pick(["return undefined", "// fixed", "const x = 2", "throw new Error(\"unexpected\")", "if (cond)"]),
|
||||
...(maybe(0.2) ? { replaceAll: true } : {}),
|
||||
},
|
||||
}),
|
||||
// write
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "write",
|
||||
input: {
|
||||
filePath: `/opencode/${pick(FILE_NAMES)}`,
|
||||
content: pick([
|
||||
"export const value = 42\n",
|
||||
"// generated\nexport default {}\n",
|
||||
"TODO: fill in\n",
|
||||
'{\n "version": "0.0.1"\n}\n',
|
||||
]),
|
||||
},
|
||||
}),
|
||||
// apply_patch
|
||||
() => {
|
||||
const file = `/opencode/${pick(FILE_NAMES)}`
|
||||
const before = pick(["return null", "const x = 1", "// old"])
|
||||
const after = pick(["return undefined", "const x = 2", "// new"])
|
||||
return {
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "apply_patch",
|
||||
input: {
|
||||
patchText: `*** Begin Patch\n*** Update File: ${file}\n@@\n- ${before}\n+ ${after}\n*** End Patch\n`,
|
||||
},
|
||||
}
|
||||
},
|
||||
// read
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "read",
|
||||
input: {
|
||||
filePath: `/opencode/${pick(FILE_NAMES)}`,
|
||||
...(maybe(0.3) ? { offset: int(1, 50) } : {}),
|
||||
...(maybe(0.3) ? { limit: int(10, 200) } : {}),
|
||||
},
|
||||
}),
|
||||
// grep
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "grep",
|
||||
input: {
|
||||
pattern: pick(PATTERNS),
|
||||
...(maybe(0.5) ? { path: pick(DIR_NAMES) } : {}),
|
||||
...(maybe(0.5) ? { include: pick(["*.ts", "*.tsx", "*.{ts,tsx}", "*.md", "*.json"]) } : {}),
|
||||
},
|
||||
}),
|
||||
// glob
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "glob",
|
||||
input: {
|
||||
pattern: pick(GLOBS),
|
||||
...(maybe(0.3) ? { path: pick(DIR_NAMES) } : {}),
|
||||
},
|
||||
}),
|
||||
// bash
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "bash",
|
||||
input: {
|
||||
command: pick(COMMANDS),
|
||||
description: pick(["List files", "Show status", "Print working dir", "Run tests", "Lint code"]),
|
||||
...(maybe(0.2) ? { timeout: int(1000, 60000) } : {}),
|
||||
},
|
||||
}),
|
||||
// todowrite
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "todowrite",
|
||||
input: {
|
||||
todos: Array.from({ length: int(1, 4) }, () => ({
|
||||
content: pick([
|
||||
"Investigate failing test",
|
||||
"Refactor layer composition",
|
||||
"Add typecheck step",
|
||||
"Update docs",
|
||||
"Bump dependencies",
|
||||
]),
|
||||
status: pick(["pending", "in_progress", "completed", "cancelled"]),
|
||||
priority: pick(["high", "medium", "low"]),
|
||||
})),
|
||||
},
|
||||
}),
|
||||
// webfetch
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "webfetch",
|
||||
input: {
|
||||
url: pick(WEB_URLS),
|
||||
...(maybe(0.4) ? { format: pick(["markdown", "text", "html"]) } : {}),
|
||||
...(maybe(0.2) ? { timeout: int(5, 60) } : {}),
|
||||
},
|
||||
}),
|
||||
// websearch
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "websearch",
|
||||
input: {
|
||||
query: pick(SEARCH_QUERIES),
|
||||
...(maybe(0.3) ? { numResults: int(1, 10) } : {}),
|
||||
...(maybe(0.3) ? { type: pick(["auto", "fast", "deep"]) } : {}),
|
||||
},
|
||||
}),
|
||||
// lsp
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "lsp",
|
||||
input: {
|
||||
file: `/opencode/${pick(FILE_NAMES)}`,
|
||||
action: pick(["definition", "references", "hover", "documentSymbol", "implementation"]),
|
||||
...(maybe(0.5) ? { position: { line: int(0, 100), character: int(0, 80) } } : {}),
|
||||
},
|
||||
}),
|
||||
// task
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "task",
|
||||
input: {
|
||||
description: pick(["explore feature", "audit dependency graph", "find usage"]),
|
||||
prompt: pick([
|
||||
"Look through src/ and summarize the entry points.",
|
||||
"Find all call sites for `Bus.publish` and explain what they publish.",
|
||||
]),
|
||||
subagent_type: pick(SUBAGENTS),
|
||||
},
|
||||
}),
|
||||
// task_status
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "task_status",
|
||||
input: {
|
||||
task_id: `task-${int(1, 50).toString(36)}`,
|
||||
},
|
||||
}),
|
||||
// plan
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "plan",
|
||||
input: {
|
||||
summary: pick([
|
||||
"Refactor the layer graph to remove cycles",
|
||||
"Add diagnostic logging then propose a fix",
|
||||
"Extract a helper and add tests",
|
||||
]),
|
||||
},
|
||||
}),
|
||||
// question
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "question",
|
||||
input: {
|
||||
questions: [
|
||||
{
|
||||
question: pick(["Which directory should I scaffold under?", "Approve the rename?"]),
|
||||
header: pick(["Pick a directory", "Approve rename"]),
|
||||
options: [
|
||||
{ label: "Yes", description: "Approve" },
|
||||
{ label: "No", description: "Decline" },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
// skill
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "skill",
|
||||
input: { name: pick(SKILLS) },
|
||||
}),
|
||||
// repo_clone
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "repo_clone",
|
||||
input: {
|
||||
url: pick(["https://github.com/anomalyco/opencode", "https://github.com/effect-ts/effect"]),
|
||||
...(maybe(0.4) ? { ref: pick(["main", "dev", "v1.0.0"]) } : {}),
|
||||
},
|
||||
}),
|
||||
// repo_overview
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "repo_overview",
|
||||
input: {
|
||||
...(maybe(0.5) ? { path: pick(DIR_NAMES) } : {}),
|
||||
...(maybe(0.3) ? { depth: int(1, 4) } : {}),
|
||||
},
|
||||
}),
|
||||
// invalid (sanity / fuzz)
|
||||
() => ({
|
||||
type: "tool-call",
|
||||
toolCallId: nextToolCallId(),
|
||||
toolName: "invalid",
|
||||
input: {
|
||||
tool: pick(["foo", "bar", "definitely_not_a_tool"]),
|
||||
error: pick(["unknown tool", "missing argument"]),
|
||||
},
|
||||
}),
|
||||
]
|
||||
|
||||
// ─── Step-shape generators ───────────────────────────────────────────────────
|
||||
|
||||
type Step = ReadonlyArray<
|
||||
| { type: "text"; content: string }
|
||||
| { type: "thinking"; content: string }
|
||||
| ToolCall
|
||||
>
|
||||
|
||||
const STEP_SHAPES: ReadonlyArray<() => Step> = [
|
||||
// shape 0: single text
|
||||
() => [{ type: "text", content: pick(PLAIN_TEXT) }],
|
||||
// shape 1: multi text
|
||||
() =>
|
||||
Array.from({ length: int(2, 5) }, () => ({
|
||||
type: "text" as const,
|
||||
content: pick(PLAIN_TEXT),
|
||||
})),
|
||||
// shape 2: thinking + text
|
||||
() => [
|
||||
{ type: "thinking" as const, content: pick(THINKING) },
|
||||
{ type: "text" as const, content: pick(PLAIN_TEXT) },
|
||||
],
|
||||
// shape 3: text + single tool-call
|
||||
() => [{ type: "text" as const, content: pick(PLAIN_TEXT) }, pick(TOOL_GENERATORS)()],
|
||||
// shape 4: thinking + text + tool-call
|
||||
() => [
|
||||
{ type: "thinking" as const, content: pick(THINKING) },
|
||||
{ type: "text" as const, content: pick(PLAIN_TEXT) },
|
||||
pick(TOOL_GENERATORS)(),
|
||||
],
|
||||
// shape 5: multiple tool calls in one step
|
||||
() => [
|
||||
{ type: "text" as const, content: pick(PLAIN_TEXT) },
|
||||
...Array.from({ length: int(2, 4) }, () => pick(TOOL_GENERATORS)()),
|
||||
],
|
||||
// shape 6: thinking only
|
||||
() => [{ type: "thinking" as const, content: pick(THINKING) }],
|
||||
// shape 7: long narrative (5-8 texts)
|
||||
() =>
|
||||
Array.from({ length: int(5, 8) }, () => ({
|
||||
type: "text" as const,
|
||||
content: pick(PLAIN_TEXT),
|
||||
})),
|
||||
// shape 8: solo tool-call (no preamble)
|
||||
() => [pick(TOOL_GENERATORS)()],
|
||||
]
|
||||
|
||||
// ─── Script generator ────────────────────────────────────────────────────────
|
||||
|
||||
type LLMScript = {
|
||||
steps: Step[]
|
||||
usage?: { inputTokens: number; outputTokens: number; totalTokens: number }
|
||||
finish?: "stop" | "tool-calls" | "length" | "unknown"
|
||||
}
|
||||
|
||||
function makeScript(): LLMScript {
|
||||
const step = pick(STEP_SHAPES)()
|
||||
const hasToolCall = step.some((item) => item.type === "tool-call")
|
||||
const finish: LLMScript["finish"] = hasToolCall ? "tool-calls" : pick(FINISH_REASONS)
|
||||
const inputTokens = int(20, 600)
|
||||
const outputTokens = int(4, 250)
|
||||
return {
|
||||
steps: [step],
|
||||
usage: { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens },
|
||||
finish,
|
||||
}
|
||||
}
|
||||
|
||||
const scripts: LLMScript[] = Array.from({ length: TOTAL }, makeScript)
|
||||
|
||||
// ─── Output ──────────────────────────────────────────────────────────────────
|
||||
|
||||
// ─── User-driven consumption loop ────────────────────────────────────────────
|
||||
//
|
||||
// `enqueueLLM` only fills the backend queue. To actually consume responses we
|
||||
// need user messages that drive `doStream` calls. Each user turn typically
|
||||
// consumes 1-2 scripts (one for the main response, one if a tool-call comes
|
||||
// back and the loop fetches a follow-up; title-gen on the first turn eats one
|
||||
// more). We send TURNS user messages so the total consumed roughly matches
|
||||
// the queue size; the default "Simulation mock response." catches any
|
||||
// remainder.
|
||||
|
||||
const PROMPTS = [
|
||||
"Walk me through the project layout briefly.",
|
||||
"Find any TODOs in the codebase.",
|
||||
"Refactor the small helper in src/util/log.ts.",
|
||||
"Open src/index.ts and explain the entrypoint.",
|
||||
"Search for `console.log` usage.",
|
||||
"Run the test suite.",
|
||||
"Patch the greeting in src/greeting.ts to say Hello.",
|
||||
"Write a small note in docs/getting-started.md.",
|
||||
"Show me recent git activity.",
|
||||
"What's the LSP status?",
|
||||
"Plan a fix for the failing test.",
|
||||
"Summarize the changes so far.",
|
||||
"Look up the npm registry entry for `effect`.",
|
||||
"Search the web for `lsp protocol initialize`.",
|
||||
"Outline the next refactor step.",
|
||||
]
|
||||
|
||||
const TURNS = Math.ceil(TOTAL / 2) // ~2 LLM calls per turn on average
|
||||
|
||||
const userActions = Array.from({ length: TURNS }, (_, i) => [
|
||||
{ type: "typeText", text: PROMPTS[i % PROMPTS.length]! },
|
||||
{ type: "pressEnter" },
|
||||
// Small wait so the prompt loop can drain LLM calls before the next input.
|
||||
{ type: "wait", ms: 60 },
|
||||
]).flat()
|
||||
|
||||
const script = {
|
||||
_comment: `Generated by 09_diverse_llm_responses.gen.ts. ${TOTAL} LLM responses with diverse step shapes, tool calls (every registered tool kind), and finish reasons. Followed by ${TURNS} user turns that drive the backend to consume them. Seed: 0x${SEED.toString(16)}. Re-run the generator to regenerate.`,
|
||||
actions: [
|
||||
// Enter Build mode (skip default plan agent).
|
||||
{ type: "pressKey", key: "x", modifiers: { ctrl: true } },
|
||||
{ type: "pressKey", key: "b" },
|
||||
{ type: "enqueueLLM", scripts },
|
||||
...userActions,
|
||||
],
|
||||
}
|
||||
|
||||
writeFileSync(OUTPUT_PATH, JSON.stringify(script, null, 2) + "\n")
|
||||
|
||||
// ─── Summary printed for the developer ───────────────────────────────────────
|
||||
|
||||
const byFinish = scripts.reduce(
|
||||
(acc, s) => {
|
||||
const k = s.finish ?? "stop"
|
||||
acc[k] = (acc[k] ?? 0) + 1
|
||||
return acc
|
||||
},
|
||||
{} as Record<string, number>,
|
||||
)
|
||||
|
||||
const byTool: Record<string, number> = {}
|
||||
let totalToolCalls = 0
|
||||
for (const s of scripts) {
|
||||
for (const step of s.steps) {
|
||||
for (const item of step) {
|
||||
if (item.type === "tool-call") {
|
||||
byTool[item.toolName] = (byTool[item.toolName] ?? 0) + 1
|
||||
totalToolCalls++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Wrote ${OUTPUT_PATH}`)
|
||||
console.log(`Scripts: ${scripts.length}`)
|
||||
console.log(`Finish reasons:`, byFinish)
|
||||
console.log(`Tool calls (${totalToolCalls} total):`, byTool)
|
||||
File diff suppressed because it is too large
Load diff
888
packages/opencode/test/testing/simulation/scripts/generate.ts
Normal file
888
packages/opencode/test/testing/simulation/scripts/generate.ts
Normal file
|
|
@ -0,0 +1,888 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Simulation script generator.
|
||||
*
|
||||
* Builds a single simulation script (JSON) that:
|
||||
* 1. Enters Build mode (ctrl+x, b).
|
||||
* 2. Optionally seeds the simulated filesystem with a starter set of files.
|
||||
* 3. Queues N LLM responses chosen by step-shape and tool-kind weights.
|
||||
* 4. Drives the TUI through M user turns to consume those LLM responses.
|
||||
*
|
||||
* Critically: a stateful FS model tracks which paths exist at each step, so
|
||||
* `apply_patch`/`edit`/`read` tool calls only ever target real files. `write`
|
||||
* tool calls add new files into the model so later patches can target them.
|
||||
*
|
||||
* Usage:
|
||||
* bun test/testing/simulation/scripts/generate.ts [options]
|
||||
*
|
||||
* Options:
|
||||
* --out <path> Output JSON path. Default: ./generated.json
|
||||
* --total <n> Total LLM scripts to queue. Default: 1000
|
||||
* --turns <n> Number of user turns. Default: ceil(total / 2)
|
||||
* --seed <n> RNG seed (hex or decimal). Default: 0x09abcdef
|
||||
* --tools <list> Comma-separated tool kinds to include. Default: all.
|
||||
* --shapes <list> Comma-separated step shapes. Default: all.
|
||||
* --weight tool=<n> Override weight for a single tool. Repeatable.
|
||||
* --weight shape=<n> Override weight for a single shape. Repeatable.
|
||||
* --seed-files Pre-seed common files into the simulated FS.
|
||||
* Default: true. Pass --no-seed-files to disable.
|
||||
* --enable-titles Enqueue +1 short script per turn for title gen.
|
||||
* Default: true. Pass --no-enable-titles to disable.
|
||||
*
|
||||
* Examples:
|
||||
*
|
||||
* # Default: 1000 scripts, 500 turns, balanced tool mix
|
||||
* bun generate.ts --out diverse.json
|
||||
*
|
||||
* # Patch-heavy run: only apply_patch + read, 200 turns
|
||||
* bun generate.ts --total 600 --turns 200 \
|
||||
* --tools apply_patch,read --out patches.json
|
||||
*
|
||||
* # Bias towards bash + apply_patch
|
||||
* bun generate.ts --weight tool.apply_patch=10 --weight tool.bash=10
|
||||
*/
|
||||
|
||||
import { writeFileSync } from "fs"
|
||||
import path from "path"
|
||||
|
||||
// ─── CLI ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface CliOptions {
|
||||
out: string
|
||||
total: number
|
||||
turns: number
|
||||
seed: number
|
||||
tools: Set<string> | null
|
||||
shapes: Set<string> | null
|
||||
toolWeights: Record<string, number>
|
||||
shapeWeights: Record<string, number>
|
||||
seedFiles: boolean
|
||||
enableTitles: boolean
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): CliOptions {
|
||||
const out: CliOptions = {
|
||||
out: "./generated.json",
|
||||
total: 1000,
|
||||
turns: -1,
|
||||
seed: 0x09abcdef,
|
||||
tools: null,
|
||||
shapes: null,
|
||||
toolWeights: {},
|
||||
shapeWeights: {},
|
||||
seedFiles: true,
|
||||
enableTitles: true,
|
||||
}
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i]!
|
||||
const next = () => {
|
||||
const v = argv[++i]
|
||||
if (v === undefined) throw new Error(`${a} requires a value`)
|
||||
return v
|
||||
}
|
||||
switch (a) {
|
||||
case "--out":
|
||||
out.out = next()
|
||||
break
|
||||
case "--total":
|
||||
out.total = Number(next())
|
||||
break
|
||||
case "--turns":
|
||||
out.turns = Number(next())
|
||||
break
|
||||
case "--seed": {
|
||||
const v = next()
|
||||
out.seed = v.startsWith("0x") ? parseInt(v, 16) : Number(v)
|
||||
break
|
||||
}
|
||||
case "--tools":
|
||||
out.tools = new Set(
|
||||
next()
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
)
|
||||
break
|
||||
case "--shapes":
|
||||
out.shapes = new Set(
|
||||
next()
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
)
|
||||
break
|
||||
case "--weight": {
|
||||
// `tool.apply_patch=5` or `shape.toolCall=3`
|
||||
const value = next()
|
||||
const eq = value.indexOf("=")
|
||||
if (eq < 0) throw new Error(`--weight expects key=number, got ${value}`)
|
||||
const [k, v] = [value.slice(0, eq), Number(value.slice(eq + 1))]
|
||||
if (k.startsWith("tool.")) out.toolWeights[k.slice(5)] = v
|
||||
else if (k.startsWith("shape.")) out.shapeWeights[k.slice(6)] = v
|
||||
else throw new Error(`--weight key must start with tool. or shape., got ${k}`)
|
||||
break
|
||||
}
|
||||
case "--seed-files":
|
||||
out.seedFiles = true
|
||||
break
|
||||
case "--no-seed-files":
|
||||
out.seedFiles = false
|
||||
break
|
||||
case "--enable-titles":
|
||||
out.enableTitles = true
|
||||
break
|
||||
case "--no-enable-titles":
|
||||
out.enableTitles = false
|
||||
break
|
||||
case "--help":
|
||||
case "-h":
|
||||
printHelp()
|
||||
process.exit(0)
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${a}`)
|
||||
}
|
||||
}
|
||||
if (out.turns < 0) out.turns = Math.ceil(out.total / 2)
|
||||
return out
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Simulation script generator.
|
||||
|
||||
Usage:
|
||||
bun generate.ts [options]
|
||||
|
||||
Options:
|
||||
--out <path> Output JSON path (default ./generated.json)
|
||||
--total <n> Total LLM scripts to queue (default 1000)
|
||||
--turns <n> User turns (default ceil(total/2))
|
||||
--seed <n> RNG seed (hex or decimal, default 0x09abcdef)
|
||||
--tools <list> Comma-separated tool kinds to include
|
||||
--shapes <list> Comma-separated step shapes
|
||||
--weight tool.<id>=<n> Override weight for a tool
|
||||
--weight shape.<id>=<n> Override weight for a step shape
|
||||
--seed-files Pre-seed starter files (default on)
|
||||
--no-seed-files Skip pre-seeding files
|
||||
--enable-titles Pad queue for title-gen calls (default on)
|
||||
--no-enable-titles Don't pad for title-gen
|
||||
|
||||
Available tool kinds:
|
||||
apply_patch, edit, write, read, grep, glob, bash, todowrite, webfetch,
|
||||
websearch, lsp, task, task_status, plan, question, skill, repo_clone,
|
||||
repo_overview, invalid
|
||||
|
||||
Available step shapes:
|
||||
text, multiText, thinkText, textToolCall, thinkTextToolCall, multiToolCall,
|
||||
thinkOnly, longNarrative, soloToolCall
|
||||
`)
|
||||
}
|
||||
|
||||
// ─── Seeded RNG (mulberry32) ─────────────────────────────────────────────────
|
||||
|
||||
function mulberry32(seed: number) {
|
||||
let state = seed >>> 0
|
||||
return () => {
|
||||
state = (state + 0x6d2b79f5) >>> 0
|
||||
let t = state
|
||||
t = Math.imul(t ^ (t >>> 15), t | 1)
|
||||
t ^= t + Math.imul(t ^ (t >>> 7), t | 61)
|
||||
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Filesystem model ────────────────────────────────────────────────────────
|
||||
//
|
||||
// Tracks the file paths that we've materialized so far. `apply_patch`/`edit`/
|
||||
// `read` only target paths in this set. `write` adds to the set. The pre-seed
|
||||
// step populates the initial set so the first random scripts have something to
|
||||
// patch.
|
||||
|
||||
class FsModel {
|
||||
private paths = new Set<string>()
|
||||
private seeded: { path: string; content: string }[] = []
|
||||
|
||||
seed(path: string, content: string) {
|
||||
this.paths.add(path)
|
||||
this.seeded.push({ path, content })
|
||||
}
|
||||
add(path: string) {
|
||||
this.paths.add(path)
|
||||
}
|
||||
has(path: string) {
|
||||
return this.paths.has(path)
|
||||
}
|
||||
all(): string[] {
|
||||
return [...this.paths]
|
||||
}
|
||||
seededWrites() {
|
||||
return this.seeded
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Vocab ───────────────────────────────────────────────────────────────────
|
||||
|
||||
const SEED_FILES: { path: string; content: string }[] = [
|
||||
{ path: "src/index.ts", content: "export function main() {\n return 0\n}\n" },
|
||||
{ path: "src/server.ts", content: "export const server = {\n start() {},\n}\n" },
|
||||
{ path: "src/util/log.ts", content: "export function log(msg: string) {\n console.log(msg)\n}\n" },
|
||||
{ path: "src/util/string.ts", content: "export const upper = (s: string) => s.toUpperCase()\n" },
|
||||
{ path: "src/feature/auth.ts", content: "export const auth = { user: null }\n" },
|
||||
{ path: "src/feature/cart.ts", content: "export const cart: string[] = []\n" },
|
||||
{ path: "src/lib/db.ts", content: "export const db = { query() { return null } }\n" },
|
||||
{ path: "src/lib/cache.ts", content: "export const cache = new Map<string, unknown>()\n" },
|
||||
{ path: "src/api/users.ts", content: "export const users = []\n" },
|
||||
{ path: "test/index.test.ts", content: "import { test } from 'bun:test'\ntest('ok', () => {})\n" },
|
||||
{ path: "test/auth.test.ts", content: "import { test } from 'bun:test'\ntest('auth', () => {})\n" },
|
||||
{ path: "README.md", content: "# project\n\nDescription.\n" },
|
||||
{ path: "docs/api.md", content: "# api\n\nDocs.\n" },
|
||||
{ path: "docs/getting-started.md", content: "# getting started\n\nWelcome.\n" },
|
||||
{ path: "package.json", content: '{\n "name": "project",\n "version": "0.1.0"\n}\n' },
|
||||
]
|
||||
|
||||
const NEW_FILE_CANDIDATES = [
|
||||
"src/feature/checkout.tsx",
|
||||
"src/components/Button.tsx",
|
||||
"src/components/Modal.tsx",
|
||||
"src/api/orders.ts",
|
||||
"test/cart.test.ts",
|
||||
"scripts/build.ts",
|
||||
"config/eslint.json",
|
||||
"CHANGELOG.md",
|
||||
"src/lib/format.ts",
|
||||
"src/lib/clock.ts",
|
||||
]
|
||||
|
||||
const PATTERNS = [
|
||||
"TODO",
|
||||
"FIXME",
|
||||
"console\\.log",
|
||||
"function\\s+\\w+",
|
||||
"import .* from",
|
||||
"export const",
|
||||
"throw new Error",
|
||||
"async\\s+function",
|
||||
"class\\s+\\w+",
|
||||
"interface\\s+\\w+",
|
||||
]
|
||||
|
||||
const GLOBS = ["**/*.ts", "**/*.tsx", "src/**/*.ts", "test/**/*.test.ts", "**/*.{ts,tsx}", "**/*.md", "**/*.json"]
|
||||
|
||||
const COMMANDS = [
|
||||
"ls -la",
|
||||
"pwd",
|
||||
"cat README.md",
|
||||
"wc -l src/index.ts",
|
||||
"git status",
|
||||
"git log --oneline -5",
|
||||
"git diff --stat",
|
||||
"bun install",
|
||||
"bun test",
|
||||
"bun run build",
|
||||
"rg TODO",
|
||||
"echo 'hello'",
|
||||
"date",
|
||||
]
|
||||
|
||||
const WEB_URLS = [
|
||||
"https://example.com/api/data",
|
||||
"https://docs.opencode.ai/configuration",
|
||||
"https://github.com/anomalyco/opencode",
|
||||
"https://api.openai.com/v1/models",
|
||||
"https://registry.npmjs.org/effect",
|
||||
"https://nodejs.org/api/fs.html",
|
||||
]
|
||||
|
||||
const SEARCH_QUERIES = [
|
||||
"rust async error handling",
|
||||
"effect-ts schema validation",
|
||||
"react server components",
|
||||
"typescript discriminated unions",
|
||||
"bun sqlite performance",
|
||||
"lsp protocol initialize",
|
||||
"git rebase squash workflow",
|
||||
]
|
||||
|
||||
const SKILLS = ["customize-opencode", "effect", "improve-codebase-architecture", "gmail"]
|
||||
const SUBAGENTS = ["explore", "general"]
|
||||
|
||||
const PLAIN_TEXT = [
|
||||
"Looking at the code now.",
|
||||
"Let me inspect the relevant files.",
|
||||
"I'll start by reading the entrypoint.",
|
||||
"Checking the test suite for related coverage.",
|
||||
"Tracing the call chain through the layer composition.",
|
||||
"This looks like a missing dependency in the layer graph.",
|
||||
"I'll add a small helper to factor out the duplication.",
|
||||
"Renaming the symbol everywhere it's used.",
|
||||
"Bumping the version in package.json.",
|
||||
"Adding a changelog entry.",
|
||||
"Running the formatter.",
|
||||
"Re-running the typecheck.",
|
||||
"All clean — no type errors.",
|
||||
"Tests pass locally.",
|
||||
"Drafting the PR description.",
|
||||
]
|
||||
|
||||
const THINKING = [
|
||||
"Need to figure out which layer is missing the dependency.",
|
||||
"The error stack points at instance-state.ts — likely missing InstanceRef.",
|
||||
"Best to check if the cache is being invalidated correctly.",
|
||||
"Tradeoff: inline vs extract helper. Inline is shorter.",
|
||||
"Race condition seems likely given the await boundary.",
|
||||
"Looking at the diff to spot the regression.",
|
||||
"Need to make sure the test asserts the post-condition.",
|
||||
]
|
||||
|
||||
const FINISH_REASONS: ("stop" | "tool-calls" | "length" | "unknown")[] = [
|
||||
"stop",
|
||||
"tool-calls",
|
||||
"stop",
|
||||
"tool-calls",
|
||||
"stop",
|
||||
"length",
|
||||
"stop",
|
||||
"unknown",
|
||||
]
|
||||
|
||||
const PROMPTS = [
|
||||
"Walk me through the project layout briefly.",
|
||||
"Find any TODOs in the codebase.",
|
||||
"Refactor the small helper in src/util/log.ts.",
|
||||
"Open src/index.ts and explain the entrypoint.",
|
||||
"Search for `console.log` usage.",
|
||||
"Run the test suite.",
|
||||
"Patch the greeting in src/greeting.ts to say Hello.",
|
||||
"Write a small note in docs/getting-started.md.",
|
||||
"Show me recent git activity.",
|
||||
"What's the LSP status?",
|
||||
"Plan a fix for the failing test.",
|
||||
"Summarize the changes so far.",
|
||||
"Look up the npm registry entry for `effect`.",
|
||||
"Search the web for `lsp protocol initialize`.",
|
||||
"Outline the next refactor step.",
|
||||
]
|
||||
|
||||
// ─── Tool generators ─────────────────────────────────────────────────────────
|
||||
|
||||
type ToolCall = {
|
||||
type: "tool-call"
|
||||
toolCallId: string
|
||||
toolName: string
|
||||
input: Record<string, unknown>
|
||||
}
|
||||
|
||||
interface Helpers {
|
||||
rand: () => number
|
||||
pick: <T>(items: readonly T[]) => T
|
||||
int: (min: number, max: number) => number
|
||||
maybe: (p: number) => boolean
|
||||
nextToolCallId: () => string
|
||||
fs: FsModel
|
||||
}
|
||||
|
||||
const TOOL_KINDS = [
|
||||
"apply_patch",
|
||||
"edit",
|
||||
"write",
|
||||
"read",
|
||||
"grep",
|
||||
"glob",
|
||||
"bash",
|
||||
"todowrite",
|
||||
"webfetch",
|
||||
"websearch",
|
||||
"lsp",
|
||||
"task",
|
||||
"task_status",
|
||||
"plan",
|
||||
"question",
|
||||
"skill",
|
||||
"repo_clone",
|
||||
"repo_overview",
|
||||
"invalid",
|
||||
] as const
|
||||
type ToolKind = (typeof TOOL_KINDS)[number]
|
||||
|
||||
// Tool generators take the helpers + return a ToolCall, or null if the tool
|
||||
// can't be produced right now (e.g. apply_patch needs an existing file).
|
||||
type ToolGen = (h: Helpers) => ToolCall | null
|
||||
|
||||
const TOOL_GENERATORS: Record<ToolKind, ToolGen> = {
|
||||
apply_patch: (h) => {
|
||||
const existing = h.fs.all()
|
||||
if (existing.length === 0) return null
|
||||
const file = `/opencode/${h.pick(existing)}`
|
||||
const before = h.pick(["export const", "return", "function", "import"])
|
||||
const after = h.pick(["export default", "return undefined", "async function", "import type"])
|
||||
return {
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "apply_patch",
|
||||
input: {
|
||||
patchText: `*** Begin Patch\n*** Update File: ${file}\n@@\n- ${before}\n+ ${after}\n*** End Patch\n`,
|
||||
},
|
||||
}
|
||||
},
|
||||
edit: (h) => {
|
||||
const existing = h.fs.all()
|
||||
if (existing.length === 0) return null
|
||||
return {
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "edit",
|
||||
input: {
|
||||
filePath: `/opencode/${h.pick(existing)}`,
|
||||
oldString: h.pick(["return null", "// TODO", "const x = 1", "if (true)"]),
|
||||
newString: h.pick(["return undefined", "// fixed", "const x = 2", "if (cond)"]),
|
||||
...(h.maybe(0.2) ? { replaceAll: true } : {}),
|
||||
},
|
||||
}
|
||||
},
|
||||
write: (h) => {
|
||||
const candidates = NEW_FILE_CANDIDATES.filter((p) => !h.fs.has(p))
|
||||
const file = candidates.length > 0 ? h.pick(candidates) : h.pick(NEW_FILE_CANDIDATES)
|
||||
const content = h.pick([
|
||||
"export const value = 42\n",
|
||||
"// generated\nexport default {}\n",
|
||||
"TODO: fill in\n",
|
||||
'{\n "version": "0.0.1"\n}\n',
|
||||
])
|
||||
// Track the write so future apply_patch/edit can target it.
|
||||
h.fs.add(file)
|
||||
return {
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "write",
|
||||
input: { filePath: `/opencode/${file}`, content },
|
||||
}
|
||||
},
|
||||
read: (h) => {
|
||||
const existing = h.fs.all()
|
||||
if (existing.length === 0) return null
|
||||
return {
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "read",
|
||||
input: {
|
||||
filePath: `/opencode/${h.pick(existing)}`,
|
||||
...(h.maybe(0.3) ? { offset: h.int(1, 50) } : {}),
|
||||
...(h.maybe(0.3) ? { limit: h.int(10, 200) } : {}),
|
||||
},
|
||||
}
|
||||
},
|
||||
grep: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "grep",
|
||||
input: {
|
||||
pattern: h.pick(PATTERNS),
|
||||
...(h.maybe(0.5) ? { path: h.pick(["src", "src/feature", "src/util", "test", "docs"]) } : {}),
|
||||
...(h.maybe(0.5) ? { include: h.pick(["*.ts", "*.tsx", "*.{ts,tsx}", "*.md"]) } : {}),
|
||||
},
|
||||
}),
|
||||
glob: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "glob",
|
||||
input: {
|
||||
pattern: h.pick(GLOBS),
|
||||
...(h.maybe(0.3) ? { path: h.pick(["src", "test", "docs"]) } : {}),
|
||||
},
|
||||
}),
|
||||
bash: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "bash",
|
||||
input: {
|
||||
command: h.pick(COMMANDS),
|
||||
description: h.pick(["List files", "Show status", "Print working dir", "Run tests"]),
|
||||
...(h.maybe(0.2) ? { timeout: h.int(1000, 60000) } : {}),
|
||||
},
|
||||
}),
|
||||
todowrite: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "todowrite",
|
||||
input: {
|
||||
todos: Array.from({ length: h.int(1, 4) }, () => ({
|
||||
content: h.pick([
|
||||
"Investigate failing test",
|
||||
"Refactor layer composition",
|
||||
"Add typecheck step",
|
||||
"Update docs",
|
||||
"Bump dependencies",
|
||||
]),
|
||||
status: h.pick(["pending", "in_progress", "completed", "cancelled"]),
|
||||
priority: h.pick(["high", "medium", "low"]),
|
||||
})),
|
||||
},
|
||||
}),
|
||||
webfetch: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "webfetch",
|
||||
input: {
|
||||
url: h.pick(WEB_URLS),
|
||||
...(h.maybe(0.4) ? { format: h.pick(["markdown", "text", "html"]) } : {}),
|
||||
...(h.maybe(0.2) ? { timeout: h.int(5, 60) } : {}),
|
||||
},
|
||||
}),
|
||||
websearch: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "websearch",
|
||||
input: {
|
||||
query: h.pick(SEARCH_QUERIES),
|
||||
...(h.maybe(0.3) ? { numResults: h.int(1, 10) } : {}),
|
||||
...(h.maybe(0.3) ? { type: h.pick(["auto", "fast", "deep"]) } : {}),
|
||||
},
|
||||
}),
|
||||
lsp: (h) => {
|
||||
const existing = h.fs.all().filter((p) => /\.(ts|tsx|js|jsx)$/.test(p))
|
||||
const file = existing.length > 0 ? `/opencode/${h.pick(existing)}` : "/opencode/src/index.ts"
|
||||
return {
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "lsp",
|
||||
input: {
|
||||
file,
|
||||
action: h.pick(["definition", "references", "hover", "documentSymbol", "implementation"]),
|
||||
...(h.maybe(0.5) ? { position: { line: h.int(0, 100), character: h.int(0, 80) } } : {}),
|
||||
},
|
||||
}
|
||||
},
|
||||
task: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "task",
|
||||
input: {
|
||||
description: h.pick(["explore feature", "audit dependency graph", "find usage"]),
|
||||
prompt: h.pick([
|
||||
"Look through src/ and summarize the entry points.",
|
||||
"Find all call sites for `Bus.publish` and explain what they publish.",
|
||||
]),
|
||||
subagent_type: h.pick(SUBAGENTS),
|
||||
},
|
||||
}),
|
||||
task_status: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "task_status",
|
||||
input: { task_id: `task-${h.int(1, 50).toString(36)}` },
|
||||
}),
|
||||
plan: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "plan",
|
||||
input: {
|
||||
summary: h.pick([
|
||||
"Refactor the layer graph to remove cycles",
|
||||
"Add diagnostic logging then propose a fix",
|
||||
"Extract a helper and add tests",
|
||||
]),
|
||||
},
|
||||
}),
|
||||
question: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "question",
|
||||
input: {
|
||||
questions: [
|
||||
{
|
||||
question: h.pick(["Which directory should I scaffold under?", "Approve the rename?"]),
|
||||
header: h.pick(["Pick a directory", "Approve rename"]),
|
||||
options: [
|
||||
{ label: "Yes", description: "Approve" },
|
||||
{ label: "No", description: "Decline" },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
skill: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "skill",
|
||||
input: { name: h.pick(SKILLS) },
|
||||
}),
|
||||
repo_clone: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "repo_clone",
|
||||
input: {
|
||||
url: h.pick(["https://github.com/anomalyco/opencode", "https://github.com/effect-ts/effect"]),
|
||||
...(h.maybe(0.4) ? { ref: h.pick(["main", "dev", "v1.0.0"]) } : {}),
|
||||
},
|
||||
}),
|
||||
repo_overview: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "repo_overview",
|
||||
input: {
|
||||
...(h.maybe(0.5) ? { path: h.pick(["src", "test", "docs"]) } : {}),
|
||||
...(h.maybe(0.3) ? { depth: h.int(1, 4) } : {}),
|
||||
},
|
||||
}),
|
||||
invalid: (h) => ({
|
||||
type: "tool-call",
|
||||
toolCallId: h.nextToolCallId(),
|
||||
toolName: "invalid",
|
||||
input: {
|
||||
tool: h.pick(["foo", "bar", "definitely_not_a_tool"]),
|
||||
error: h.pick(["unknown tool", "missing argument"]),
|
||||
},
|
||||
}),
|
||||
}
|
||||
|
||||
const DEFAULT_TOOL_WEIGHTS: Record<ToolKind, number> = {
|
||||
apply_patch: 4,
|
||||
edit: 3,
|
||||
write: 3,
|
||||
read: 4,
|
||||
grep: 3,
|
||||
glob: 3,
|
||||
bash: 2,
|
||||
todowrite: 2,
|
||||
webfetch: 2,
|
||||
websearch: 2,
|
||||
lsp: 2,
|
||||
task: 2,
|
||||
task_status: 1,
|
||||
plan: 2,
|
||||
question: 1,
|
||||
skill: 1,
|
||||
repo_clone: 1,
|
||||
repo_overview: 1,
|
||||
invalid: 1,
|
||||
}
|
||||
|
||||
// ─── Step shapes ─────────────────────────────────────────────────────────────
|
||||
|
||||
type StepItem =
|
||||
| { type: "text"; content: string }
|
||||
| { type: "thinking"; content: string }
|
||||
| ToolCall
|
||||
|
||||
type ShapeKind =
|
||||
| "text"
|
||||
| "multiText"
|
||||
| "thinkText"
|
||||
| "textToolCall"
|
||||
| "thinkTextToolCall"
|
||||
| "multiToolCall"
|
||||
| "thinkOnly"
|
||||
| "longNarrative"
|
||||
| "soloToolCall"
|
||||
|
||||
interface ShapeContext {
|
||||
h: Helpers
|
||||
pickTool: () => ToolCall | null
|
||||
}
|
||||
|
||||
const SHAPE_BUILDERS: Record<ShapeKind, (ctx: ShapeContext) => StepItem[] | null> = {
|
||||
text: ({ h }) => [{ type: "text", content: h.pick(PLAIN_TEXT) }],
|
||||
multiText: ({ h }) =>
|
||||
Array.from({ length: h.int(2, 5) }, () => ({ type: "text" as const, content: h.pick(PLAIN_TEXT) })),
|
||||
thinkText: ({ h }) => [
|
||||
{ type: "thinking", content: h.pick(THINKING) },
|
||||
{ type: "text", content: h.pick(PLAIN_TEXT) },
|
||||
],
|
||||
textToolCall: ({ h, pickTool }) => {
|
||||
const tc = pickTool()
|
||||
if (!tc) return null
|
||||
return [{ type: "text", content: h.pick(PLAIN_TEXT) }, tc]
|
||||
},
|
||||
thinkTextToolCall: ({ h, pickTool }) => {
|
||||
const tc = pickTool()
|
||||
if (!tc) return null
|
||||
return [
|
||||
{ type: "thinking", content: h.pick(THINKING) },
|
||||
{ type: "text", content: h.pick(PLAIN_TEXT) },
|
||||
tc,
|
||||
]
|
||||
},
|
||||
multiToolCall: ({ h, pickTool }) => {
|
||||
const count = h.int(2, 4)
|
||||
const tcs: ToolCall[] = []
|
||||
for (let i = 0; i < count; i++) {
|
||||
const tc = pickTool()
|
||||
if (tc) tcs.push(tc)
|
||||
}
|
||||
if (tcs.length === 0) return null
|
||||
return [{ type: "text", content: h.pick(PLAIN_TEXT) }, ...tcs]
|
||||
},
|
||||
thinkOnly: ({ h }) => [{ type: "thinking", content: h.pick(THINKING) }],
|
||||
longNarrative: ({ h }) =>
|
||||
Array.from({ length: h.int(5, 8) }, () => ({ type: "text" as const, content: h.pick(PLAIN_TEXT) })),
|
||||
soloToolCall: ({ pickTool }) => {
|
||||
const tc = pickTool()
|
||||
if (!tc) return null
|
||||
return [tc]
|
||||
},
|
||||
}
|
||||
|
||||
const DEFAULT_SHAPE_WEIGHTS: Record<ShapeKind, number> = {
|
||||
text: 3,
|
||||
multiText: 2,
|
||||
thinkText: 2,
|
||||
textToolCall: 5,
|
||||
thinkTextToolCall: 3,
|
||||
multiToolCall: 2,
|
||||
thinkOnly: 1,
|
||||
longNarrative: 2,
|
||||
soloToolCall: 2,
|
||||
}
|
||||
|
||||
// ─── Weighted picker ─────────────────────────────────────────────────────────
|
||||
|
||||
function weightedPicker<K extends string>(weights: Record<K, number>, rand: () => number) {
|
||||
const entries = Object.entries(weights) as [K, number][]
|
||||
const positive = entries.filter(([, w]) => w > 0)
|
||||
if (positive.length === 0) throw new Error("All weights are zero")
|
||||
const total = positive.reduce((sum, [, w]) => sum + w, 0)
|
||||
return () => {
|
||||
let r = rand() * total
|
||||
for (const [k, w] of positive) {
|
||||
r -= w
|
||||
if (r <= 0) return k
|
||||
}
|
||||
return positive[positive.length - 1]![0]
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
function main() {
|
||||
const opts = parseArgs(process.argv.slice(2))
|
||||
|
||||
const rand = mulberry32(opts.seed)
|
||||
const pick = <T,>(items: readonly T[]): T => items[Math.floor(rand() * items.length)]!
|
||||
const int = (min: number, max: number) => min + Math.floor(rand() * (max - min + 1))
|
||||
const maybe = (p: number) => rand() < p
|
||||
|
||||
let toolCallSeq = 0
|
||||
const nextToolCallId = () => `tc-${(++toolCallSeq).toString(36)}`
|
||||
|
||||
const fs = new FsModel()
|
||||
if (opts.seedFiles) for (const f of SEED_FILES) fs.seed(f.path, f.content)
|
||||
|
||||
const helpers: Helpers = { rand, pick, int, maybe, nextToolCallId, fs }
|
||||
|
||||
const enabledTools = (TOOL_KINDS as readonly ToolKind[]).filter(
|
||||
(t) => !opts.tools || opts.tools.has(t),
|
||||
)
|
||||
if (enabledTools.length === 0) throw new Error("--tools filter excluded every tool")
|
||||
const toolWeights = Object.fromEntries(
|
||||
enabledTools.map((t) => [t, opts.toolWeights[t] ?? DEFAULT_TOOL_WEIGHTS[t]]),
|
||||
) as Record<ToolKind, number>
|
||||
const pickToolKind = weightedPicker(toolWeights, rand)
|
||||
|
||||
const enabledShapes = (Object.keys(DEFAULT_SHAPE_WEIGHTS) as ShapeKind[]).filter(
|
||||
(s) => !opts.shapes || opts.shapes.has(s),
|
||||
)
|
||||
if (enabledShapes.length === 0) throw new Error("--shapes filter excluded every shape")
|
||||
const shapeWeights = Object.fromEntries(
|
||||
enabledShapes.map((s) => [s, opts.shapeWeights[s] ?? DEFAULT_SHAPE_WEIGHTS[s]]),
|
||||
) as Record<ShapeKind, number>
|
||||
const pickShapeKind = weightedPicker(shapeWeights, rand)
|
||||
|
||||
// Resolve a tool, retrying up to N times if a generator returns null (e.g.
|
||||
// apply_patch with no existing files). If every retry fails, fall back to
|
||||
// `text` step shape via returning null upstream.
|
||||
const tryPickTool = (): ToolCall | null => {
|
||||
for (let i = 0; i < 6; i++) {
|
||||
const tc = TOOL_GENERATORS[pickToolKind() as ToolKind](helpers)
|
||||
if (tc) return tc
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
type LLMScript = {
|
||||
steps: StepItem[][]
|
||||
usage?: { inputTokens: number; outputTokens: number; totalTokens: number }
|
||||
finish?: "stop" | "tool-calls" | "length" | "unknown"
|
||||
}
|
||||
|
||||
function makeScript(): LLMScript {
|
||||
// Try up to 4 different shapes; if none yields a step (e.g. all
|
||||
// tool-shape variants returned null), fall back to a `text` step.
|
||||
let step: StepItem[] | null = null
|
||||
for (let i = 0; i < 4 && !step; i++) {
|
||||
const shape = pickShapeKind() as ShapeKind
|
||||
step = SHAPE_BUILDERS[shape]({ h: helpers, pickTool: tryPickTool })
|
||||
}
|
||||
if (!step) step = [{ type: "text", content: pick(PLAIN_TEXT) }]
|
||||
const hasToolCall = step.some((item) => item.type === "tool-call")
|
||||
const finish: LLMScript["finish"] = hasToolCall ? "tool-calls" : pick(FINISH_REASONS)
|
||||
const inputTokens = int(20, 600)
|
||||
const outputTokens = int(4, 250)
|
||||
return {
|
||||
steps: [step],
|
||||
usage: { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens },
|
||||
finish,
|
||||
}
|
||||
}
|
||||
|
||||
// Title-gen padding: each user turn the small-model gets invoked too. To
|
||||
// keep the user-visible follow-up text from falling back to the default
|
||||
// "Simulation mock response.", pad the queue by `turns` extra short scripts.
|
||||
const titlePadding = opts.enableTitles ? opts.turns : 0
|
||||
const scripts: LLMScript[] = Array.from({ length: opts.total + titlePadding }, makeScript)
|
||||
|
||||
// ─── User actions ──────────────────────────────────────────────────────────
|
||||
|
||||
const userActions = Array.from({ length: opts.turns }, (_, i) => [
|
||||
{ type: "typeText", text: PROMPTS[i % PROMPTS.length]! },
|
||||
{ type: "pressEnter" },
|
||||
{ type: "wait", ms: 60 },
|
||||
]).flat()
|
||||
|
||||
// Seed-file writes go FIRST so the FS exists before user turns run.
|
||||
const seedWrites = fs.seededWrites().map((f) => ({
|
||||
type: "writeFile",
|
||||
path: f.path,
|
||||
content: f.content,
|
||||
}))
|
||||
|
||||
const script = {
|
||||
_comment: `Generated by generate.ts. Seed: 0x${opts.seed.toString(16)}. Total LLM scripts: ${opts.total} (+${titlePadding} title padding = ${scripts.length}). Turns: ${opts.turns}. Tools enabled: ${enabledTools.length}. Pre-seeded files: ${fs.seededWrites().length}.`,
|
||||
actions: [
|
||||
{ type: "pressKey", key: "x", modifiers: { ctrl: true } },
|
||||
{ type: "pressKey", key: "b" },
|
||||
...seedWrites,
|
||||
{ type: "enqueueLLM", scripts },
|
||||
...userActions,
|
||||
],
|
||||
}
|
||||
|
||||
const outPath = path.resolve(opts.out)
|
||||
writeFileSync(outPath, JSON.stringify(script, null, 2) + "\n")
|
||||
|
||||
// ─── Summary ───────────────────────────────────────────────────────────────
|
||||
|
||||
const byFinish: Record<string, number> = {}
|
||||
for (const s of scripts) {
|
||||
const k = s.finish ?? "stop"
|
||||
byFinish[k] = (byFinish[k] ?? 0) + 1
|
||||
}
|
||||
const byTool: Record<string, number> = {}
|
||||
let totalToolCalls = 0
|
||||
for (const s of scripts) {
|
||||
for (const step of s.steps) {
|
||||
for (const item of step) {
|
||||
if (item.type === "tool-call") {
|
||||
byTool[item.toolName] = (byTool[item.toolName] ?? 0) + 1
|
||||
totalToolCalls++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`Wrote ${outPath}`)
|
||||
console.log(`Total scripts: ${scripts.length} (${opts.total} primary + ${titlePadding} title padding)`)
|
||||
console.log(`User turns: ${opts.turns}`)
|
||||
console.log(`Seeded files: ${fs.seededWrites().length}`)
|
||||
console.log(`Finish reasons:`, byFinish)
|
||||
console.log(`Tool calls (${totalToolCalls} total):`, byTool)
|
||||
}
|
||||
|
||||
main()
|
||||
File diff suppressed because it is too large
Load diff
8343
packages/opencode/test/testing/simulation/scripts/generated.json
Normal file
8343
packages/opencode/test/testing/simulation/scripts/generated.json
Normal file
File diff suppressed because it is too large
Load diff
306
packages/opencode/test/testing/simulation/scripts/run.ts
Normal file
306
packages/opencode/test/testing/simulation/scripts/run.ts
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Simulation script runner.
|
||||
*
|
||||
* Drives a simulation script against a running `bun dev simulate` process via
|
||||
* the simulation MCP HTTP endpoint. Loads the script, steps through it in
|
||||
* configurable chunks, and checks the in-memory log buffer after each chunk.
|
||||
* Stops on the first error (configurable level) and prints the failing entries
|
||||
* plus the most recent step that was executed.
|
||||
*
|
||||
* Usage:
|
||||
* bun test/testing/simulation/scripts/run.ts <script.json> [options]
|
||||
*
|
||||
* Options:
|
||||
* --mcp <url> MCP endpoint. Default: http://127.0.0.1:43110/mcp
|
||||
* --chunk <n> Actions per step batch. Default: 3
|
||||
* --max-steps <n> Hard cap on step calls. Default: unlimited.
|
||||
* --level <lvl> Stop on entries at or above this level.
|
||||
* One of DEBUG | INFO | WARN | ERROR. Default: ERROR.
|
||||
* --message-includes <s> Only stop when matching message substring.
|
||||
* --service-includes <s> Only stop when matching tag.service substring.
|
||||
* --reset Reset simulation state + restart TUI before loading.
|
||||
* --no-reset Skip the reset/restart (default).
|
||||
* --keep-going Don't stop on errors; print them and continue.
|
||||
* --quiet Suppress per-batch progress output.
|
||||
* --json Emit a single JSON summary at the end.
|
||||
* --check-every <n> Check logs only every N batches. Default: 1.
|
||||
*
|
||||
* Example:
|
||||
* bun test/testing/simulation/scripts/run.ts patches.json \
|
||||
* --reset --chunk 3 --level ERROR
|
||||
*/
|
||||
|
||||
interface Options {
|
||||
scriptPath: string
|
||||
mcpUrl: string
|
||||
chunk: number
|
||||
maxSteps: number
|
||||
level: "DEBUG" | "INFO" | "WARN" | "ERROR"
|
||||
messageIncludes?: string
|
||||
serviceIncludes?: string
|
||||
reset: boolean
|
||||
keepGoing: boolean
|
||||
quiet: boolean
|
||||
json: boolean
|
||||
checkEvery: number
|
||||
}
|
||||
|
||||
function parseArgs(argv: string[]): Options {
|
||||
if (argv.length === 0 || argv[0] === "--help" || argv[0] === "-h") {
|
||||
printHelp()
|
||||
process.exit(0)
|
||||
}
|
||||
const out: Options = {
|
||||
scriptPath: argv[0]!,
|
||||
mcpUrl: "http://127.0.0.1:43110/mcp",
|
||||
chunk: 3,
|
||||
maxSteps: -1,
|
||||
level: "ERROR",
|
||||
reset: false,
|
||||
keepGoing: false,
|
||||
quiet: false,
|
||||
json: false,
|
||||
checkEvery: 1,
|
||||
}
|
||||
for (let i = 1; i < argv.length; i++) {
|
||||
const a = argv[i]!
|
||||
const next = () => {
|
||||
const v = argv[++i]
|
||||
if (v === undefined) throw new Error(`${a} requires a value`)
|
||||
return v
|
||||
}
|
||||
switch (a) {
|
||||
case "--mcp":
|
||||
out.mcpUrl = next()
|
||||
break
|
||||
case "--chunk":
|
||||
out.chunk = Number(next())
|
||||
break
|
||||
case "--max-steps":
|
||||
out.maxSteps = Number(next())
|
||||
break
|
||||
case "--level": {
|
||||
const v = next().toUpperCase()
|
||||
if (!["DEBUG", "INFO", "WARN", "ERROR"].includes(v)) {
|
||||
throw new Error(`--level must be DEBUG|INFO|WARN|ERROR, got ${v}`)
|
||||
}
|
||||
out.level = v as Options["level"]
|
||||
break
|
||||
}
|
||||
case "--message-includes":
|
||||
out.messageIncludes = next()
|
||||
break
|
||||
case "--service-includes":
|
||||
out.serviceIncludes = next()
|
||||
break
|
||||
case "--reset":
|
||||
out.reset = true
|
||||
break
|
||||
case "--no-reset":
|
||||
out.reset = false
|
||||
break
|
||||
case "--keep-going":
|
||||
out.keepGoing = true
|
||||
break
|
||||
case "--quiet":
|
||||
out.quiet = true
|
||||
break
|
||||
case "--json":
|
||||
out.json = true
|
||||
break
|
||||
case "--check-every":
|
||||
out.checkEvery = Number(next())
|
||||
break
|
||||
case "--help":
|
||||
case "-h":
|
||||
printHelp()
|
||||
process.exit(0)
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${a}`)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Simulation script runner.
|
||||
|
||||
Usage:
|
||||
bun run.ts <script.json> [options]
|
||||
|
||||
Options:
|
||||
--mcp <url> MCP endpoint (default http://127.0.0.1:43110/mcp)
|
||||
--chunk <n> Actions per step batch (default 3)
|
||||
--max-steps <n> Hard cap on step calls (default unlimited)
|
||||
--level <lvl> Stop level: DEBUG|INFO|WARN|ERROR (default ERROR)
|
||||
--message-includes <s> Only stop when message includes substring
|
||||
--service-includes <s> Only stop when tag.service includes substring
|
||||
--reset Reset sim state + restart TUI before load
|
||||
--no-reset Skip reset (default)
|
||||
--keep-going Don't stop on errors; continue to end
|
||||
--quiet Suppress per-batch progress
|
||||
--json Emit JSON summary at the end
|
||||
--check-every <n> Check logs every N batches (default 1)
|
||||
`)
|
||||
}
|
||||
|
||||
// ─── MCP client ──────────────────────────────────────────────────────────────
|
||||
|
||||
let rpcId = 0
|
||||
|
||||
async function mcpCall(mcpUrl: string, name: string, args: Record<string, unknown> = {}) {
|
||||
const response = await fetch(mcpUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
accept: "application/json, text/event-stream",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
jsonrpc: "2.0",
|
||||
id: ++rpcId,
|
||||
method: "tools/call",
|
||||
params: { name, arguments: args },
|
||||
}),
|
||||
})
|
||||
if (!response.ok) throw new Error(`${name}: HTTP ${response.status}`)
|
||||
const text = await response.text()
|
||||
// The MCP server may respond with a single JSON line or with SSE-style
|
||||
// chunks. Find the first `{"result"` line either way.
|
||||
const jsonLine = text
|
||||
.split("\n")
|
||||
.map((line) => line.replace(/^data:\s*/, ""))
|
||||
.find((line) => line.trim().startsWith('{"result"') || line.trim().startsWith('{"error"'))
|
||||
if (!jsonLine) throw new Error(`${name}: no JSON-RPC response in:\n${text.slice(0, 500)}`)
|
||||
const envelope = JSON.parse(jsonLine)
|
||||
if (envelope.error) {
|
||||
throw new Error(`${name}: ${envelope.error.message ?? JSON.stringify(envelope.error)}`)
|
||||
}
|
||||
const content = envelope.result?.content?.[0]?.text
|
||||
if (typeof content !== "string") {
|
||||
throw new Error(`${name}: unexpected MCP envelope:\n${JSON.stringify(envelope).slice(0, 500)}`)
|
||||
}
|
||||
// Tool result `text` payloads are JSON-encoded.
|
||||
return JSON.parse(content)
|
||||
}
|
||||
|
||||
// ─── Logic ───────────────────────────────────────────────────────────────────
|
||||
|
||||
interface LogEntry {
|
||||
time: string
|
||||
level: "DEBUG" | "INFO" | "WARN" | "ERROR"
|
||||
tags: Record<string, unknown>
|
||||
message: string
|
||||
}
|
||||
|
||||
async function run(opts: Options) {
|
||||
const log = (msg: string) => {
|
||||
if (!opts.quiet) console.log(msg)
|
||||
}
|
||||
|
||||
if (opts.reset) {
|
||||
log("Resetting simulation state...")
|
||||
await mcpCall(opts.mcpUrl, "simulation_control_reset")
|
||||
log("Restarting TUI...")
|
||||
await mcpCall(opts.mcpUrl, "simulation_restart")
|
||||
}
|
||||
|
||||
log("Clearing log buffer...")
|
||||
await mcpCall(opts.mcpUrl, "simulation_log_clear")
|
||||
|
||||
log(`Loading script: ${opts.scriptPath}`)
|
||||
const loaded = await mcpCall(opts.mcpUrl, "simulation_script_load", {
|
||||
path: opts.scriptPath,
|
||||
replace: true,
|
||||
})
|
||||
log(` id=${loaded.id} total=${loaded.total}`)
|
||||
|
||||
const total = loaded.total as number
|
||||
let batches = 0
|
||||
let cursor = 0
|
||||
let stopReason: "completed" | "max-steps" | "error" = "completed"
|
||||
let stoppingEntries: LogEntry[] | undefined
|
||||
let stoppingExecuted: unknown[] | undefined
|
||||
|
||||
while (cursor < total) {
|
||||
if (opts.maxSteps > 0 && batches >= opts.maxSteps) {
|
||||
stopReason = "max-steps"
|
||||
break
|
||||
}
|
||||
|
||||
const step = await mcpCall(opts.mcpUrl, "simulation_script_step", { steps: opts.chunk })
|
||||
cursor = step.state.cursor as number
|
||||
batches++
|
||||
if (!opts.quiet) {
|
||||
const lastKinds = (step.executed as { type: string }[]).map((a) => a.type).join(",")
|
||||
log(`batch ${batches}: cursor ${cursor}/${total} — ${lastKinds}`)
|
||||
}
|
||||
|
||||
if (batches % opts.checkEvery !== 0 && cursor < total) continue
|
||||
|
||||
const logResp = await mcpCall(opts.mcpUrl, "simulation_log_get", {
|
||||
level: opts.level,
|
||||
...(opts.messageIncludes ? { messageIncludes: opts.messageIncludes } : {}),
|
||||
...(opts.serviceIncludes ? { serviceIncludes: opts.serviceIncludes } : {}),
|
||||
})
|
||||
const entries = (logResp.entries ?? []) as LogEntry[]
|
||||
if (entries.length > 0) {
|
||||
if (opts.keepGoing) {
|
||||
if (!opts.quiet) {
|
||||
console.log(
|
||||
` ⚠ ${entries.length} ${opts.level} entries — continuing (--keep-going)`,
|
||||
)
|
||||
}
|
||||
// Clear so we only see new ones in subsequent batches.
|
||||
await mcpCall(opts.mcpUrl, "simulation_log_clear")
|
||||
} else {
|
||||
stopReason = "error"
|
||||
stoppingEntries = entries
|
||||
stoppingExecuted = step.executed
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const status = await mcpCall(opts.mcpUrl, "simulation_script_status")
|
||||
|
||||
if (opts.json) {
|
||||
process.stdout.write(
|
||||
JSON.stringify(
|
||||
{
|
||||
stopReason,
|
||||
batches,
|
||||
cursor,
|
||||
total,
|
||||
status: status.status,
|
||||
stoppingEntries,
|
||||
stoppingExecuted,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
) + "\n",
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
console.log()
|
||||
console.log(`Stopped: ${stopReason}`)
|
||||
console.log(`Batches: ${batches}`)
|
||||
console.log(`Cursor: ${cursor}/${total}`)
|
||||
if (stopReason === "error" && stoppingEntries) {
|
||||
console.log()
|
||||
console.log(`${stoppingEntries.length} ${opts.level} entries — failing batch executed:`)
|
||||
console.log(JSON.stringify(stoppingExecuted, null, 2))
|
||||
console.log()
|
||||
console.log(`${opts.level} entries:`)
|
||||
console.log(JSON.stringify(stoppingEntries, null, 2))
|
||||
process.exitCode = 1
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await run(parseArgs(process.argv.slice(2)))
|
||||
} catch (err) {
|
||||
console.error(err instanceof Error ? err.message : err)
|
||||
process.exit(1)
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue