fix(core): expose models.dev modes as models (#34714)
This commit is contained in:
Aiden Cline
GitHub
parent
2324a63fc6
commit
f8626865b9
2 changed files with 203 additions and 54 deletions
|
|
@ -11,7 +11,7 @@ function released(date: string) {
|
|||
return Number.isFinite(time) ? time : 0
|
||||
}
|
||||
|
||||
function cost(input: ModelsDev.Model["cost"]) {
|
||||
function cost(input: ModelsDev.Model["cost"]): ModelV2Info["cost"] {
|
||||
const base = {
|
||||
input: input?.input ?? 0,
|
||||
output: input?.output ?? 0,
|
||||
|
|
@ -20,25 +20,57 @@ function cost(input: ModelsDev.Model["cost"]) {
|
|||
write: input?.cache_write ?? 0,
|
||||
},
|
||||
}
|
||||
if (!input?.context_over_200k) return [base]
|
||||
return [
|
||||
base,
|
||||
{
|
||||
tier: {
|
||||
type: "context" as const,
|
||||
size: 200_000,
|
||||
},
|
||||
input: input.context_over_200k.input,
|
||||
output: input.context_over_200k.output,
|
||||
...(input?.tiers?.map((item) => ({
|
||||
tier: item.tier,
|
||||
input: item.input,
|
||||
output: item.output,
|
||||
cache: {
|
||||
read: input.context_over_200k.cache_read ?? 0,
|
||||
write: input.context_over_200k.cache_write ?? 0,
|
||||
read: item.cache_read ?? 0,
|
||||
write: item.cache_write ?? 0,
|
||||
},
|
||||
},
|
||||
})) ?? []),
|
||||
...(input?.context_over_200k
|
||||
? [
|
||||
{
|
||||
tier: {
|
||||
type: "context" as const,
|
||||
size: 200_000,
|
||||
},
|
||||
input: input.context_over_200k.input,
|
||||
output: input.context_over_200k.output,
|
||||
cache: {
|
||||
read: input.context_over_200k.cache_read ?? 0,
|
||||
write: input.context_over_200k.cache_write ?? 0,
|
||||
},
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]
|
||||
}
|
||||
|
||||
function variants(model: ModelsDev.Model, packageName: string | undefined): ModelV2Info["variants"] {
|
||||
function mergeCost(base: ModelV2Info["cost"], override: ModelsDev.Model["cost"] | undefined) {
|
||||
if (!override) return base
|
||||
const next = cost(override)
|
||||
const [baseDefault, ...baseTiers] = base
|
||||
const [nextDefault, ...nextTiers] = next
|
||||
const tierKey = (item: ModelV2Info["cost"][number]) => `${item.tier?.type ?? "base"}:${item.tier?.size ?? 0}`
|
||||
const merge = (left: ModelV2Info["cost"][number], right: ModelV2Info["cost"][number]) => ({
|
||||
...left,
|
||||
...right,
|
||||
tier: right.tier ?? left.tier,
|
||||
cache: { ...left.cache, ...right.cache },
|
||||
})
|
||||
const tiers = new Map(baseTiers.map((item) => [tierKey(item), item]))
|
||||
for (const item of nextTiers) {
|
||||
const current = tiers.get(tierKey(item))
|
||||
tiers.set(tierKey(item), current ? merge(current, item) : item)
|
||||
}
|
||||
return [merge(baseDefault ?? { input: 0, output: 0, cache: { read: 0, write: 0 } }, nextDefault), ...tiers.values()]
|
||||
}
|
||||
|
||||
function reasoningVariants(model: ModelsDev.Model, packageName: string | undefined): ModelV2Info["variants"] {
|
||||
const result = new Map<ModelV2.VariantID, ModelV2Info["variants"][number]>()
|
||||
if (packageName === "@ai-sdk/openai" || packageName === "@ai-sdk/openai-compatible") {
|
||||
const option = model.reasoning_options?.find((option) => option.type === "effort")
|
||||
|
|
@ -56,17 +88,13 @@ function variants(model: ModelsDev.Model, packageName: string | undefined): Mode
|
|||
})
|
||||
}
|
||||
}
|
||||
for (const [id, item] of Object.entries(model.experimental?.modes ?? {})) {
|
||||
const variantID = ModelV2.VariantID.make(id)
|
||||
result.set(variantID, {
|
||||
id: variantID,
|
||||
headers: { ...(item.provider?.headers ?? {}) },
|
||||
body: { ...(item.provider?.body ?? {}) },
|
||||
})
|
||||
}
|
||||
return [...result.values()]
|
||||
}
|
||||
|
||||
function modeName(model: ModelsDev.Model, mode: string) {
|
||||
return `${model.name} ${mode.charAt(0).toUpperCase()}${mode.slice(1)}`
|
||||
}
|
||||
|
||||
function mergeVariants(model: ModelV2Info, next: ModelV2Info["variants"]) {
|
||||
const existing = new Map(model.variants.map((variant) => [variant.id, variant]))
|
||||
const nextIDs = new Set(next.map((variant) => variant.id))
|
||||
|
|
@ -76,6 +104,50 @@ function mergeVariants(model: ModelV2Info, next: ModelV2Info["variants"]) {
|
|||
]
|
||||
}
|
||||
|
||||
function applyModel(
|
||||
draft: ModelV2Info,
|
||||
model: ModelsDev.Model,
|
||||
input: {
|
||||
readonly name?: string
|
||||
readonly cost?: ModelV2Info["cost"]
|
||||
readonly request?: NonNullable<NonNullable<ModelsDev.Model["experimental"]>["modes"]>[string]["provider"]
|
||||
readonly variants?: ModelV2Info["variants"]
|
||||
} = {},
|
||||
) {
|
||||
draft.name = input.name ?? model.name
|
||||
draft.family = model.family ? ModelV2.Family.make(model.family) : undefined
|
||||
draft.api = model.provider?.npm
|
||||
? {
|
||||
id: ModelV2.ID.make(model.id),
|
||||
type: "aisdk",
|
||||
package: model.provider.npm,
|
||||
url: model.provider.api,
|
||||
}
|
||||
: {
|
||||
id: ModelV2.ID.make(model.id),
|
||||
type: "native",
|
||||
url: model.provider?.api,
|
||||
settings: {},
|
||||
}
|
||||
draft.capabilities = {
|
||||
tools: model.tool_call,
|
||||
input: [...(model.modalities?.input ?? [])],
|
||||
output: [...(model.modalities?.output ?? [])],
|
||||
}
|
||||
mergeVariants(draft, input.variants ?? [])
|
||||
draft.time.released = released(model.release_date)
|
||||
draft.cost = input.cost ?? cost(model.cost)
|
||||
draft.status = model.status ?? "active"
|
||||
draft.enabled = true
|
||||
draft.limit = {
|
||||
context: model.limit.context,
|
||||
input: model.limit.input,
|
||||
output: model.limit.output,
|
||||
}
|
||||
Object.assign(draft.request.headers, input.request?.headers ?? {})
|
||||
Object.assign(draft.request.body, input.request?.body ?? {})
|
||||
}
|
||||
|
||||
export const ModelsDevPlugin = define({
|
||||
id: "models-dev",
|
||||
effect: Effect.fn(function* (ctx) {
|
||||
|
|
@ -120,39 +192,19 @@ export const ModelsDevPlugin = define({
|
|||
})
|
||||
|
||||
for (const model of Object.values(item.models)) {
|
||||
const modelID = ModelV2.ID.make(model.id)
|
||||
catalog.model.update(providerID, modelID, (draft) => {
|
||||
draft.name = model.name
|
||||
draft.family = model.family ? ModelV2.Family.make(model.family) : undefined
|
||||
draft.api = model.provider?.npm
|
||||
? {
|
||||
id: draft.api.id,
|
||||
type: "aisdk",
|
||||
package: model.provider?.npm,
|
||||
url: model.provider.api,
|
||||
}
|
||||
: {
|
||||
id: draft.api.id,
|
||||
type: "native",
|
||||
url: model.provider?.api,
|
||||
settings: {},
|
||||
}
|
||||
draft.capabilities = {
|
||||
tools: model.tool_call,
|
||||
input: [...(model.modalities?.input ?? [])],
|
||||
output: [...(model.modalities?.output ?? [])],
|
||||
}
|
||||
mergeVariants(draft, variants(model, model.provider?.npm ?? item.npm))
|
||||
draft.time.released = released(model.release_date)
|
||||
draft.cost = cost(model.cost)
|
||||
draft.status = model.status ?? "active"
|
||||
draft.enabled = true
|
||||
draft.limit = {
|
||||
context: model.limit.context,
|
||||
input: model.limit.input,
|
||||
output: model.limit.output,
|
||||
}
|
||||
})
|
||||
const baseCost = cost(model.cost)
|
||||
const variants = reasoningVariants(model, model.provider?.npm ?? item.npm)
|
||||
catalog.model.update(providerID, model.id, (draft) => applyModel(draft, model, { cost: baseCost, variants }))
|
||||
for (const [mode, options] of Object.entries(model.experimental?.modes ?? {})) {
|
||||
catalog.model.update(providerID, `${model.id}-${mode}`, (draft) =>
|
||||
applyModel(draft, model, {
|
||||
name: modeName(model, mode),
|
||||
cost: mergeCost(baseCost, options.cost),
|
||||
request: options.provider,
|
||||
variants,
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}),
|
||||
|
|
|
|||
|
|
@ -32,6 +32,103 @@ const layer = Layer.mergeAll(catalog.pipe(Layer.provide(connections)), integrati
|
|||
const it = testEffect(layer)
|
||||
|
||||
describe("ModelsDevPlugin", () => {
|
||||
it.effect("projects models.dev modes as separate models instead of variants", () =>
|
||||
Effect.gen(function* () {
|
||||
const integrations = yield* Integration.Service
|
||||
const catalog = yield* Catalog.Service
|
||||
const models = ModelsDev.Service.of({
|
||||
get: () =>
|
||||
Effect.succeed({
|
||||
acme: {
|
||||
id: "acme",
|
||||
name: "Acme",
|
||||
env: [],
|
||||
npm: "@ai-sdk/openai-compatible",
|
||||
api: "https://api.acme.test/v1",
|
||||
models: {
|
||||
"gpt-5.4": {
|
||||
id: "gpt-5.4",
|
||||
name: "GPT-5.4",
|
||||
family: "gpt",
|
||||
release_date: "2026-01-01",
|
||||
attachment: false,
|
||||
reasoning: true,
|
||||
temperature: true,
|
||||
tool_call: true,
|
||||
cost: {
|
||||
input: 2.5,
|
||||
output: 15,
|
||||
tiers: [
|
||||
{
|
||||
tier: { type: "context", size: 272_000 },
|
||||
input: 3,
|
||||
output: 18,
|
||||
cache_read: 0.25,
|
||||
},
|
||||
],
|
||||
context_over_200k: { input: 5, output: 22.5, cache_read: 0.5 },
|
||||
},
|
||||
limit: { context: 1_050_000, input: 922_000, output: 128_000 },
|
||||
experimental: {
|
||||
modes: {
|
||||
fast: {
|
||||
cost: { input: 5, output: 30, cache_read: 0.5 },
|
||||
provider: {
|
||||
headers: { "x-mode": "fast" },
|
||||
body: { service_tier: "priority" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} satisfies Record<string, ModelsDev.Provider>),
|
||||
refresh: () => Effect.void,
|
||||
})
|
||||
|
||||
yield* ModelsDevPlugin.effect(
|
||||
host({
|
||||
catalog: catalogHost(catalog),
|
||||
integration: integrationHost(integrations),
|
||||
}),
|
||||
).pipe(Effect.provideService(ModelsDev.Service, models))
|
||||
|
||||
const providerID = ProviderV2.ID.make("acme")
|
||||
const base = yield* catalog.model.get(providerID, ModelV2.ID.make("gpt-5.4"))
|
||||
const fast = yield* catalog.model.get(providerID, ModelV2.ID.make("gpt-5.4-fast"))
|
||||
|
||||
expect(base?.variants).toEqual([])
|
||||
expect(base?.request.body).toEqual({})
|
||||
expect(fast).toMatchObject({
|
||||
id: "gpt-5.4-fast",
|
||||
providerID: "acme",
|
||||
name: "GPT-5.4 Fast",
|
||||
api: { id: "gpt-5.4" },
|
||||
request: {
|
||||
headers: { "x-mode": "fast" },
|
||||
body: { service_tier: "priority" },
|
||||
},
|
||||
variants: [],
|
||||
})
|
||||
expect(fast?.cost).toEqual([
|
||||
{ input: 5, output: 30, cache: { read: 0.5, write: 0 } },
|
||||
{
|
||||
tier: { type: "context", size: 272_000 },
|
||||
input: 3,
|
||||
output: 18,
|
||||
cache: { read: 0.25, write: 0 },
|
||||
},
|
||||
{
|
||||
tier: { type: "context", size: 200_000 },
|
||||
input: 5,
|
||||
output: 22.5,
|
||||
cache: { read: 0.5, write: 0 },
|
||||
},
|
||||
])
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("registers key methods for providers with environment variables", () =>
|
||||
Effect.acquireUseRelease(
|
||||
Effect.sync(() => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue