test(core): align models dev snapshot fixtures

This commit is contained in:
Dax Raad 2026-07-09 13:09:47 -04:00
commit a2718177be
2 changed files with 190 additions and 99 deletions

View file

@ -1,12 +1,15 @@
import { describe, expect, beforeAll, beforeEach, afterAll } from "bun:test" import { describe, expect, beforeAll, beforeEach, afterAll } from "bun:test"
import { Effect, Layer, Ref, Schema } from "effect" import { Money } from "@opencode-ai/schema/money"
import { Effect, Layer, Ref } from "effect"
import { HttpClient, HttpClientResponse } from "effect/unstable/http" import { HttpClient, HttpClientResponse } from "effect/unstable/http"
import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder" import { AppNodeBuilder } from "@opencode-ai/core/effect/app-node-builder"
import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform" import { LayerNodePlatform } from "@opencode-ai/core/effect/app-node-platform"
import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { LayerNode } from "@opencode-ai/core/effect/layer-node"
import { Flag } from "@opencode-ai/core/flag/flag" import { Flag } from "@opencode-ai/core/flag/flag"
import { Global } from "@opencode-ai/core/global" import { Global } from "@opencode-ai/core/global"
import { ModelV2 } from "@opencode-ai/core/model"
import { ModelsDev } from "@opencode-ai/core/models-dev" import { ModelsDev } from "@opencode-ai/core/models-dev"
import { ProviderV2 } from "@opencode-ai/core/provider"
import { it } from "./lib/effect" import { it } from "./lib/effect"
import { readFile, rm, writeFile, utimes, mkdir } from "fs/promises" import { readFile, rm, writeFile, utimes, mkdir } from "fs/promises"
import path from "path" import path from "path"
@ -29,7 +32,7 @@ afterAll(() => {
const cacheFile = path.join(Global.Path.cache, "models.json") const cacheFile = path.join(Global.Path.cache, "models.json")
const fixture: Record<string, ModelsDev.Provider> = { const fixture = {
acme: { acme: {
id: "acme", id: "acme",
name: "Acme", name: "Acme",
@ -49,7 +52,47 @@ const fixture: Record<string, ModelsDev.Provider> = {
}, },
} }
const fixture2: Record<string, ModelsDev.Provider> = { const fixtureSnapshot = [
{
info: {
id: ProviderV2.ID.make("acme"),
name: "Acme",
package: "",
},
models: [
{
id: ModelV2.ID.make("acme-1"),
modelID: ModelV2.ID.make("acme-1"),
providerID: ProviderV2.ID.make("acme"),
name: "Acme One",
family: undefined,
package: undefined,
settings: undefined,
capabilities: { tools: true, input: [], output: [] },
variants: [],
time: { released: Date.parse("2026-01-01") },
cost: [
{
input: Money.USDPerMillionTokens.zero,
output: Money.USDPerMillionTokens.zero,
cache: {
read: Money.USDPerMillionTokens.zero,
write: Money.USDPerMillionTokens.zero,
},
},
],
status: "active",
enabled: true,
limit: { context: 128000, input: undefined, output: 8192 },
headers: undefined,
body: undefined,
},
],
environment: ["ACME_API_KEY"],
},
] satisfies readonly ModelsDev.Snapshot[]
const fixture2 = {
beta: { beta: {
id: "beta", id: "beta",
name: "Beta", name: "Beta",
@ -69,6 +112,46 @@ const fixture2: Record<string, ModelsDev.Provider> = {
}, },
} }
const fixture2Snapshot = [
{
info: {
id: ProviderV2.ID.make("beta"),
name: "Beta",
package: "",
},
models: [
{
id: ModelV2.ID.make("beta-1"),
modelID: ModelV2.ID.make("beta-1"),
providerID: ProviderV2.ID.make("beta"),
name: "Beta One",
family: undefined,
package: undefined,
settings: undefined,
capabilities: { tools: false, input: [], output: [] },
variants: [],
time: { released: Date.parse("2026-02-01") },
cost: [
{
input: Money.USDPerMillionTokens.zero,
output: Money.USDPerMillionTokens.zero,
cache: {
read: Money.USDPerMillionTokens.zero,
write: Money.USDPerMillionTokens.zero,
},
},
],
status: "active",
enabled: true,
limit: { context: 64000, input: undefined, output: 4096 },
headers: undefined,
body: undefined,
},
],
environment: ["BETA_API_KEY"],
},
] satisfies readonly ModelsDev.Snapshot[]
interface MockState { interface MockState {
body: string body: string
status: number status: number
@ -127,45 +210,7 @@ const initialState: MockState = {
} }
describe("ModelsDev Service", () => { describe("ModelsDev Service", () => {
it.effect("decodes known reasoning options", () => it.live("get() returns normalized snapshots from disk when cache file exists", () =>
Effect.sync(() => {
const result = Schema.decodeUnknownSync(ModelsDev.Model)({
id: "reasoning-model",
name: "Reasoning Model",
release_date: "2026-01-01",
attachment: false,
reasoning: true,
reasoning_options: [
{ type: "effort", values: ["low", "high"] },
{ type: "budget_tokens", min: 1024, max: 8192 },
{ type: "toggle" },
],
temperature: true,
tool_call: true,
limit: { context: 128000, output: 8192 },
})
expect(result.reasoning_options?.map((item) => item.type)).toEqual(["effort", "budget_tokens", "toggle"])
}),
)
it.effect("allows models.dev entries without temperature metadata", () =>
Effect.sync(() => {
const result = Schema.decodeUnknownSync(ModelsDev.Model)({
id: "no-temperature-model",
name: "No Temperature Model",
release_date: "2026-01-01",
attachment: false,
reasoning: false,
tool_call: true,
limit: { context: 128000, output: 8192 },
})
expect(result.temperature).toBeUndefined()
}),
)
it.live("get() returns providers from disk when cache file exists", () =>
Effect.gen(function* () { Effect.gen(function* () {
yield* writeCache(fixture) yield* writeCache(fixture)
const state = yield* Ref.make(initialState) const state = yield* Ref.make(initialState)
@ -173,7 +218,7 @@ describe("ModelsDev Service", () => {
state, state,
ModelsDev.Service.use((s) => s.get()), ModelsDev.Service.use((s) => s.get()),
) )
expect(result).toEqual(fixture) expect(result).toEqual(fixtureSnapshot)
const final = yield* Ref.get(state) const final = yield* Ref.get(state)
expect(final.calls).toEqual([]) expect(final.calls).toEqual([])
}), }),
@ -186,7 +231,7 @@ describe("ModelsDev Service", () => {
state, state,
ModelsDev.Service.use((s) => s.get()), ModelsDev.Service.use((s) => s.get()),
) )
expect(result).toEqual({}) expect(result).toEqual([])
const final = yield* Ref.get(state) const final = yield* Ref.get(state)
expect(final.calls).toEqual([]) expect(final.calls).toEqual([])
}), }),
@ -207,7 +252,7 @@ describe("ModelsDev Service", () => {
Flag.OPENCODE_DISABLE_MODELS_FETCH = true Flag.OPENCODE_DISABLE_MODELS_FETCH = true
}), }),
) )
expect(result).toEqual(fixture2) expect(result).toEqual(fixture2Snapshot)
expect(yield* Effect.promise(() => readFile(cacheFile, "utf8"))).toBe(JSON.stringify(fixture2)) expect(yield* Effect.promise(() => readFile(cacheFile, "utf8"))).toBe(JSON.stringify(fixture2))
const final = yield* Ref.get(state) const final = yield* Ref.get(state)
expect(final.calls.length).toBe(1) expect(final.calls.length).toBe(1)
@ -227,7 +272,7 @@ describe("ModelsDev Service", () => {
}) })
}), }),
) )
for (const result of results) expect(result).toEqual(fixture) for (const result of results) expect(result).toEqual(fixtureSnapshot)
}), }),
) )
@ -246,8 +291,8 @@ describe("ModelsDev Service", () => {
return { a, b } return { a, b }
}), }),
) )
expect(first.a).toEqual(fixture) expect(first.a).toEqual(fixtureSnapshot)
expect(first.b).toEqual(fixture) expect(first.b).toEqual(fixtureSnapshot)
}), }),
) )
@ -265,8 +310,8 @@ describe("ModelsDev Service", () => {
return { before, after } return { before, after }
}), }),
) )
expect(result.before).toEqual(fixture) expect(result.before).toEqual(fixtureSnapshot)
expect(result.after).toEqual(fixture2) expect(result.after).toEqual(fixture2Snapshot)
const final = yield* Ref.get(state) const final = yield* Ref.get(state)
expect(final.calls.length).toBe(1) expect(final.calls.length).toBe(1)
expect(final.calls[0].url).toContain("/api.json") expect(final.calls[0].url).toContain("/api.json")
@ -303,7 +348,7 @@ describe("ModelsDev Service", () => {
) )
const final = yield* Ref.get(state) const final = yield* Ref.get(state)
expect(final.calls.length).toBe(1) expect(final.calls.length).toBe(1)
expect(after).toEqual(fixture2) expect(after).toEqual(fixture2Snapshot)
}), }),
) )
@ -319,7 +364,7 @@ describe("ModelsDev Service", () => {
return yield* svc.get() return yield* svc.get()
}), }),
) )
expect(result).toEqual(fixture) expect(result).toEqual(fixtureSnapshot)
// retryTransient retries 5xx, so calls may be > 1. // retryTransient retries 5xx, so calls may be > 1.
const final = yield* Ref.get(state) const final = yield* Ref.get(state)
expect(final.calls.length).toBeGreaterThanOrEqual(1) expect(final.calls.length).toBeGreaterThanOrEqual(1)

View file

@ -28,66 +28,113 @@ const layer = AppNodeBuilder.build(LayerNode.group([Catalog.node, Integration.no
const it = testEffect(layer) const it = testEffect(layer)
describe("ModelsDevPlugin", () => { describe("ModelsDevPlugin", () => {
it.effect("projects models.dev modes as separate models instead of variants", () => it.effect("projects normalized models.dev snapshots into the catalog", () =>
Effect.gen(function* () { Effect.gen(function* () {
const integrations = yield* Integration.Service const integrations = yield* Integration.Service
const catalog = yield* Catalog.Service const catalog = yield* Catalog.Service
const providerID = ProviderV2.ID.make("acme")
const modelID = ModelV2.ID.make("gpt-5.4")
const models = ModelsDev.Service.of({ const models = ModelsDev.Service.of({
get: () => get: () =>
Effect.succeed({ Effect.succeed([
acme: { {
id: "acme", info: {
name: "Acme", id: providerID,
env: [], name: "Acme",
npm: "@ai-sdk/openai-compatible", package: ProviderV2.aisdk("@ai-sdk/openai-compatible"),
api: "https://api.acme.test/v1", settings: { baseURL: "https://api.acme.test/v1" },
models: { },
"gpt-5.4": { environment: [],
id: "gpt-5.4", models: [
{
id: modelID,
modelID,
providerID,
name: "GPT-5.4", name: "GPT-5.4",
family: "gpt", family: ModelV2.Family.make("gpt"),
release_date: "2026-01-01", capabilities: { tools: true, input: [], output: [] },
attachment: false, variants: [],
reasoning: true, time: { released: Date.parse("2026-01-01") },
temperature: true, cost: [
tool_call: true, {
cost: { input: Money.USDPerMillionTokens.make(2.5),
input: Money.USDPerMillionTokens.make(2.5), output: Money.USDPerMillionTokens.make(15),
output: Money.USDPerMillionTokens.make(15), cache: {
tiers: [ read: Money.USDPerMillionTokens.zero,
{ write: Money.USDPerMillionTokens.zero,
tier: { type: "context", size: 272_000 },
input: Money.USDPerMillionTokens.make(3),
output: Money.USDPerMillionTokens.make(18),
cache_read: Money.USDPerMillionTokens.make(0.25),
}, },
], },
context_over_200k: { {
tier: { type: "context", size: 272_000 },
input: Money.USDPerMillionTokens.make(3),
output: Money.USDPerMillionTokens.make(18),
cache: {
read: Money.USDPerMillionTokens.make(0.25),
write: Money.USDPerMillionTokens.zero,
},
},
{
tier: { type: "context", size: 200_000 },
input: Money.USDPerMillionTokens.make(5), input: Money.USDPerMillionTokens.make(5),
output: Money.USDPerMillionTokens.make(22.5), output: Money.USDPerMillionTokens.make(22.5),
cache_read: Money.USDPerMillionTokens.make(0.5), cache: {
}, read: Money.USDPerMillionTokens.make(0.5),
}, write: Money.USDPerMillionTokens.zero,
limit: { context: 1_050_000, input: 922_000, output: 128_000 },
experimental: {
modes: {
fast: {
cost: {
input: Money.USDPerMillionTokens.make(5),
output: Money.USDPerMillionTokens.make(30),
cache_read: Money.USDPerMillionTokens.make(0.5),
},
provider: {
headers: { "x-mode": "fast" },
body: { service_tier: "priority" },
},
}, },
}, },
}, ],
status: "active",
enabled: true,
limit: { context: 1_050_000, input: 922_000, output: 128_000 },
}, },
}, {
id: ModelV2.ID.make("gpt-5.4-fast"),
modelID,
providerID,
name: "GPT-5.4 Fast",
family: ModelV2.Family.make("gpt"),
package: ProviderV2.aisdk("@ai-sdk/openai-compatible"),
settings: { baseURL: "https://api.acme.test/v1" },
headers: { "x-mode": "fast" },
body: { service_tier: "priority" },
capabilities: { tools: true, input: [], output: [] },
variants: [],
time: { released: Date.parse("2026-01-01") },
cost: [
{
input: Money.USDPerMillionTokens.make(5),
output: Money.USDPerMillionTokens.make(30),
cache: {
read: Money.USDPerMillionTokens.make(0.5),
write: Money.USDPerMillionTokens.zero,
},
},
{
tier: { type: "context", size: 272_000 },
input: Money.USDPerMillionTokens.make(3),
output: Money.USDPerMillionTokens.make(18),
cache: {
read: Money.USDPerMillionTokens.make(0.25),
write: Money.USDPerMillionTokens.zero,
},
},
{
tier: { type: "context", size: 200_000 },
input: Money.USDPerMillionTokens.make(5),
output: Money.USDPerMillionTokens.make(22.5),
cache: {
read: Money.USDPerMillionTokens.make(0.5),
write: Money.USDPerMillionTokens.zero,
},
},
],
status: "active",
enabled: true,
limit: { context: 1_050_000, input: 922_000, output: 128_000 },
},
],
}, },
} satisfies Record<string, ModelsDev.Provider>), ] satisfies readonly ModelsDev.Snapshot[]),
refresh: () => Effect.void, refresh: () => Effect.void,
}) })
@ -98,12 +145,11 @@ describe("ModelsDevPlugin", () => {
}), }),
).pipe(Effect.provideService(ModelsDev.Service, models)) ).pipe(Effect.provideService(ModelsDev.Service, models))
const providerID = ProviderV2.ID.make("acme")
const base = yield* catalog.model.get(providerID, ModelV2.ID.make("gpt-5.4")) const base = yield* catalog.model.get(providerID, ModelV2.ID.make("gpt-5.4"))
const fast = yield* catalog.model.get(providerID, ModelV2.ID.make("gpt-5.4-fast")) const fast = yield* catalog.model.get(providerID, ModelV2.ID.make("gpt-5.4-fast"))
expect(base?.variants).toEqual([]) expect(base?.variants).toEqual([])
expect(base?.body).toEqual({}) expect(base?.body).toBeUndefined()
expect(fast).toMatchObject({ expect(fast).toMatchObject({
id: "gpt-5.4-fast", id: "gpt-5.4-fast",
modelID: "gpt-5.4", modelID: "gpt-5.4",