latest snapshot

packages/core/script/campaign-provider-catalog-race.ts

@@ -0,0 +1,76 @@
+import fs from "node:fs/promises"
+import path from "node:path"
+import { runScenario, type Delay, type Result, type Scenario } from "./provider-catalog-race-harness"
+
+const count = numberArg("--count", 64)
+const seed = numberArg("--seed", 42000)
+const out = path.resolve(stringArg("--out", "/tmp/opencode-provider-catalog-race-campaign"))
+await fs.rm(out, { recursive: true, force: true })
+await fs.mkdir(out, { recursive: true })
+
+const results: Result[] = []
+for (let index = 0; index < count; index++) {
+  const scenario = generate(seed + index, index)
+  const directory = path.join(out, `case-${String(index + 1).padStart(3, "0")}-${seed + index}`)
+  const result = await runScenario(scenario, path.join(directory, "state"))
+  await fs.mkdir(directory, { recursive: true })
+  await fs.writeFile(path.join(directory, "scenario.json"), JSON.stringify(scenario, undefined, 2) + "\n")
+  await fs.writeFile(path.join(directory, "result.json"), JSON.stringify(result, undefined, 2) + "\n")
+  results.push(result)
+  console.log(`[${index + 1}/${count}] ${scenario.id}: ${result.reproduced ? "RACE" : "ok"}`)
+}
+
+const summary = {
+  seed,
+  count,
+  reproduced: results.filter((item) => item.reproduced).length,
+  firstAttemptFailures: results.filter((item) => item.snapshots[0]?.error).length,
+  recoveredOnLaterAttempt: results.filter(
+    (item) => item.snapshots[0]?.error && item.snapshots.slice(1).some((snapshot) => snapshot.resolved),
+  ).length,
+  byDelay: Object.fromEntries(
+    (["none", "yield", "1ms", "10ms"] as Delay[]).map((delay) => [
+      delay,
+      {
+        count: results.filter((item) => item.scenario.delay === delay).length,
+        reproduced: results.filter((item) => item.scenario.delay === delay && item.reproduced).length,
+      },
+    ]),
+  ),
+}
+await fs.writeFile(path.join(out, "summary.json"), JSON.stringify(summary, undefined, 2) + "\n")
+console.log(JSON.stringify(summary, undefined, 2))
+console.log(`Artifacts: ${out}`)
+process.exitCode = summary.reproduced > 0 ? 1 : 0
+
+function generate(value: number, index: number): Scenario {
+  const random = rng(value)
+  const delays: Delay[] = ["none", "yield", "1ms", "10ms"]
+  return {
+    id: `seed-${value}`,
+    delay: delays[index % delays.length]!,
+    providerID: `provider-${index % 8}`,
+    modelID: `model-${Math.floor(random() * 8)}`,
+    configuredDefault: random() < 0.75,
+    apiKey: true,
+    disabled: random() < 0.1,
+    repeats: 3,
+  }
+}
+
+function rng(seed: number) {
+  let state = seed >>> 0
+  return () => {
+    state = (Math.imul(state, 1664525) + 1013904223) >>> 0
+    return state / 0x1_0000_0000
+  }
+}
+
+function stringArg(name: string, fallback: string) {
+  const index = process.argv.indexOf(name)
+  return index < 0 ? fallback : (process.argv[index + 1] ?? fallback)
+}
+
+function numberArg(name: string, fallback: number) {
+  return Number(stringArg(name, String(fallback)))
+}

packages/core/script/provider-catalog-race-harness.ts

@@ -0,0 +1,150 @@
+import fs from "node:fs/promises"
+import os from "node:os"
+import path from "node:path"
+import { DateTime, Effect, Layer } from "effect"
+import { AppNodeBuilder } from "../src/effect/app-node-builder"
+import { LayerNode } from "../src/effect/layer-node"
+import { Database } from "../src/database/database"
+import { EventV2 } from "../src/event"
+import { Catalog } from "../src/catalog"
+import { Location } from "../src/location"
+import { LocationServiceMap } from "../src/location-services"
+import { ModelV2 } from "../src/model"
+import { ProjectV2 } from "../src/project"
+import { ProviderV2 } from "../src/provider"
+import { AbsolutePath } from "../src/schema"
+import { SessionV2 } from "../src/session"
+import { SessionRunnerModel } from "../src/session/runner/model"
+
+export type Delay = "none" | "yield" | "1ms" | "10ms"
+
+export interface Scenario {
+  id: string
+  delay: Delay
+  providerID: string
+  modelID: string
+  configuredDefault: boolean
+  apiKey: boolean
+  disabled: boolean
+  repeats: number
+}
+
+export interface Snapshot {
+  attempt: number
+  providers: string[]
+  models: string[]
+  availableModels: string[]
+  resolved?: { providerID: string; modelID: string }
+  error?: { tag: string; message: string }
+}
+
+export interface Result {
+  scenario: Scenario
+  directory: string
+  snapshots: Snapshot[]
+  reproduced: boolean
+}
+
+const app = AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, LocationServiceMap.node]))
+
+export async function runScenario(scenario: Scenario, root?: string): Promise<Result> {
+  const directory = root ?? (await fs.mkdtemp(path.join(os.tmpdir(), "opencode-provider-race-")))
+  await fs.mkdir(directory, { recursive: true })
+  await fs.writeFile(path.join(directory, "opencode.json"), JSON.stringify(config(scenario), undefined, 2) + "\n")
+  const location = Location.Ref.make({ directory: AbsolutePath.make(directory) })
+
+  const program = Effect.gen(function* () {
+    const locations = yield* LocationServiceMap.Service
+    const context = yield* locations.contextEffect(location)
+    const output: Snapshot[] = []
+    for (let attempt = 0; attempt < scenario.repeats; attempt++) {
+      yield* delay(scenario.delay)
+      output.push(
+        yield* Effect.gen(function* () {
+          const catalog = yield* Catalog.Service
+          const providers = (yield* catalog.provider.all()).map((item) => item.id).sort()
+          const models = (yield* catalog.model.all()).map((item) => `${item.providerID}/${item.id}`).sort()
+          const availableModels = (yield* catalog.model.available())
+            .map((item) => `${item.providerID}/${item.id}`)
+            .sort()
+          const resolved = yield* SessionRunnerModel.Service.use((service) =>
+            service.resolve(session(scenario, location)),
+          ).pipe(
+            Effect.map((item) => ({ providerID: String(item.ref.providerID), modelID: String(item.ref.id) })),
+            Effect.catch((error) =>
+              Effect.succeed({
+                error: {
+                  tag: typeof error === "object" && error && "_tag" in error ? String(error._tag) : "Unknown",
+                  message: error instanceof Error ? error.message : String(error),
+                },
+              }),
+            ),
+          )
+          return {
+            attempt,
+            providers,
+            models,
+            availableModels,
+            ...(resolved && "error" in resolved ? { error: resolved.error } : { resolved }),
+          }
+        }).pipe(Effect.provide(context)),
+      )
+    }
+    return output
+  }).pipe(Effect.scoped, Effect.provide(app))
+  const snapshots = await Effect.runPromise(program)
+
+  const expected = `${scenario.providerID}/${scenario.modelID}`
+  return {
+    scenario,
+    directory,
+    snapshots,
+    reproduced: snapshots.some(
+      (item) => !item.models.includes(expected) || item.error?.tag === "SessionRunnerModel.ModelUnavailableError",
+    ),
+  }
+}
+
+function config(scenario: Scenario) {
+  return {
+    ...(scenario.configuredDefault ? { model: `${scenario.providerID}/${scenario.modelID}` } : {}),
+    providers: {
+      [scenario.providerID]: {
+        name: `Probe ${scenario.providerID}`,
+        api: { type: "aisdk", package: "@ai-sdk/openai-compatible", url: "https://probe.invalid/v1" },
+        request: { body: scenario.apiKey ? { apiKey: "probe-key" } : {} },
+        models: {
+          [scenario.modelID]: {
+            name: `Probe ${scenario.modelID}`,
+            disabled: scenario.disabled,
+            capabilities: { tools: true, input: ["text"], output: ["text"] },
+            limit: { context: 8192, output: 2048 },
+          },
+        },
+      },
+    },
+  }
+}
+
+function session(scenario: Scenario, location: Location.Ref) {
+  return SessionV2.Info.make({
+    id: SessionV2.ID.make(`ses_${scenario.id.replace(/[^a-zA-Z0-9_-]/g, "_")}`),
+    projectID: ProjectV2.ID.global,
+    title: scenario.id,
+    model: {
+      providerID: ProviderV2.ID.make(scenario.providerID),
+      id: ModelV2.ID.make(scenario.modelID),
+    },
+    cost: 0,
+    tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
+    time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) },
+    location,
+  })
+}
+
+function delay(value: Delay) {
+  if (value === "yield") return Effect.yieldNow
+  if (value === "1ms") return Effect.sleep("1 millis")
+  if (value === "10ms") return Effect.sleep("10 millis")
+  return Effect.void
+}

packages/core/script/reproduce-provider-catalog-race.ts

@@ -0,0 +1,37 @@
+import fs from "node:fs/promises"
+import path from "node:path"
+import { runScenario } from "./provider-catalog-race-harness"
+
+const out = path.resolve(process.env.OPENCODE_PROBE_OUT ?? "/tmp/opencode-provider-catalog-race-repro")
+await fs.rm(out, { recursive: true, force: true })
+await fs.mkdir(out, { recursive: true })
+
+const result = await runScenario(
+  {
+    id: "cold-immediate",
+    delay: "none",
+    providerID: "console-openai",
+    modelID: "gpt-5.6-sol",
+    configuredDefault: true,
+    apiKey: true,
+    disabled: false,
+    repeats: 2,
+  },
+  path.join(out, "state"),
+)
+
+await fs.writeFile(path.join(out, "result.json"), JSON.stringify(result, undefined, 2) + "\n")
+for (const snapshot of result.snapshots) {
+  console.log(
+    JSON.stringify({
+      attempt: snapshot.attempt,
+      providers: snapshot.providers,
+      models: snapshot.models,
+      resolved: snapshot.resolved,
+      error: snapshot.error,
+    }),
+  )
+}
+console.log(`${result.reproduced ? "REPRODUCED" : "NOT REPRODUCED"}: cold location provider catalog startup race`)
+console.log(`Artifacts: ${out}`)
+process.exitCode = result.reproduced ? 1 : 0