feat: drive reasoning variants from models.dev reasoning_options
Parse the curated reasoning_options field from models.dev api.json and use its effort values to generate reasoning variants instead of the hardcoded per-package tables, in both the v1 provider catalog and the v2 catalog plugin. - core: ModelsDev.ReasoningOption discriminated union (toggle | effort | budget_tokens); effort values stay open strings and unknown option types are tolerated since api.json is cast, not decoded - core: ReasoningVariants shared per-package effort encoder used by v1 ProviderTransform.variants and the v2 ModelsDevPlugin - core: v2 catalog generates effort variants from reasoning_options; curated experimental modes win id collisions; anthropic profile gains the effort semantic - llm: anthropic protocol supports adaptive thinking, lowers effort to output_config.effort, and sends the effort-2025-11-24 beta header - opencode: resolved Provider.Model carries capabilities.reasoningOptions (unknown types and null effort values dropped at the mapping boundary); config models accept reasoning_options; models without usable effort data fall back to the hardcoded tables unchanged Catalog-wide audit vs live api.json: 2975 models byte-identical, 38 diffs, all data correcting stale hardcoded effort lists.
This commit is contained in:
parent
c939aa04fe
commit
f7dcfc2680
15 changed files with 954 additions and 62 deletions
|
|
@ -84,7 +84,16 @@ const profiles = new Map<string, Profile>([
|
|||
]),
|
||||
},
|
||||
],
|
||||
["@ai-sdk/anthropic", { namespace: "anthropic", semantics: new Map([["thinking", "thinking"]]) }],
|
||||
[
|
||||
"@ai-sdk/anthropic",
|
||||
{
|
||||
namespace: "anthropic",
|
||||
semantics: new Map([
|
||||
["thinking", "thinking"],
|
||||
["effort", "effort"],
|
||||
]),
|
||||
},
|
||||
],
|
||||
])
|
||||
|
||||
export const namespace = (packageName: string) => profiles.get(packageName)?.namespace
|
||||
|
|
|
|||
|
|
@ -43,6 +43,30 @@ const Cost = Schema.Struct({
|
|||
),
|
||||
})
|
||||
|
||||
// models.dev curates reasoning_options per provider model as a discriminated
|
||||
// union. The shape is expected to evolve, so stay lenient where it can grow:
|
||||
// effort values are open strings (tiers like "xhigh" were added over time) and
|
||||
// budget bounds are optional. The api.json payload is cast, never decoded, so
|
||||
// option types this union doesn't know about yet can appear at runtime;
|
||||
// consumers must filter for the types they understand instead of matching
|
||||
// exhaustively.
|
||||
export const ReasoningOption = Schema.Union([
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("toggle"),
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("effort"),
|
||||
// null means the provider accepts an explicit "no reasoning" effort.
|
||||
values: Schema.Array(Schema.NullOr(Schema.String)),
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("budget_tokens"),
|
||||
min: Schema.optional(Schema.Finite),
|
||||
max: Schema.optional(Schema.Finite),
|
||||
}),
|
||||
])
|
||||
export type ReasoningOption = Schema.Schema.Type<typeof ReasoningOption>
|
||||
|
||||
export const Model = Schema.Struct({
|
||||
id: Schema.String,
|
||||
name: Schema.String,
|
||||
|
|
@ -50,6 +74,7 @@ export const Model = Schema.Struct({
|
|||
release_date: Schema.String,
|
||||
attachment: Schema.Boolean,
|
||||
reasoning: Schema.Boolean,
|
||||
reasoning_options: Schema.optional(Schema.Array(ReasoningOption)),
|
||||
temperature: Schema.Boolean,
|
||||
tool_call: Schema.Boolean,
|
||||
interleaved: Schema.optional(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import { ModelRequest } from "../model-request"
|
|||
import { ModelsDev } from "../models-dev"
|
||||
import { PluginV2 } from "../plugin"
|
||||
import { ProviderV2 } from "../provider"
|
||||
import { ReasoningVariants } from "../reasoning-variants"
|
||||
|
||||
function released(date: string) {
|
||||
const time = Date.parse(date)
|
||||
|
|
@ -39,8 +40,8 @@ function cost(input: ModelsDev.Model["cost"]) {
|
|||
]
|
||||
}
|
||||
|
||||
function variants(model: ModelsDev.Model, packageName?: string) {
|
||||
return Object.entries(model.experimental?.modes ?? {}).map(([id, item]) => {
|
||||
function variants(model: ModelsDev.Model, providerID: string, packageName?: string) {
|
||||
const modes = Object.entries(model.experimental?.modes ?? {}).map(([id, item]) => {
|
||||
const request = ModelRequest.normalizeAiSdkOptions(packageName, item.provider?.body ?? {})
|
||||
return {
|
||||
id: ModelV2.VariantID.make(id),
|
||||
|
|
@ -48,6 +49,20 @@ function variants(model: ModelsDev.Model, packageName?: string) {
|
|||
...request,
|
||||
}
|
||||
})
|
||||
// reasoning_options effort data generates effort variants with the same wire
|
||||
// encodings the v1 catalog uses; curated experimental modes win on id collision.
|
||||
const efforts = ReasoningVariants.fromOptions(
|
||||
{ npm: packageName, apiID: model.id, modelID: model.id, providerID },
|
||||
model.reasoning_options,
|
||||
)
|
||||
const fromEfforts = Object.entries(efforts ?? {})
|
||||
.filter(([id]) => !modes.some((mode) => mode.id === id))
|
||||
.map(([id, body]) => ({
|
||||
id: ModelV2.VariantID.make(id),
|
||||
headers: {},
|
||||
...ModelRequest.normalizeAiSdkOptions(packageName, body),
|
||||
}))
|
||||
return [...modes, ...fromEfforts]
|
||||
}
|
||||
|
||||
export const ModelsDevPlugin = PluginV2.define({
|
||||
|
|
@ -102,7 +117,7 @@ export const ModelsDevPlugin = PluginV2.define({
|
|||
input: [...(model.modalities?.input ?? [])],
|
||||
output: [...(model.modalities?.output ?? [])],
|
||||
}
|
||||
draft.variants = variants(model, model.provider?.npm ?? item.npm)
|
||||
draft.variants = variants(model, item.id, model.provider?.npm ?? item.npm)
|
||||
draft.time.released = released(model.release_date)
|
||||
draft.cost = cost(model.cost)
|
||||
draft.status = model.status ?? "active"
|
||||
|
|
|
|||
187
packages/core/src/reasoning-variants.ts
Normal file
187
packages/core/src/reasoning-variants.ts
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
export * as ReasoningVariants from "./reasoning-variants"
|
||||
|
||||
// Generates reasoning variants from models.dev `reasoning_options` data. The
|
||||
// data only says WHICH efforts a model supports - the wire encoding for each
|
||||
// SDK package lives here so the v1 provider catalog and the v2 catalog plugin
|
||||
// stay in lockstep. Null effort values and option types we don't understand
|
||||
// (toggle, budget_tokens, future additions) are ignored, so models without
|
||||
// usable effort data return undefined and callers fall back to their own
|
||||
// defaults.
|
||||
|
||||
// OpenAI Responses `include` value that returns the encrypted reasoning state
|
||||
// needed for stateless multi-turn reasoning (store: false). Hoisted so every
|
||||
// branch that requests it stays in lockstep.
|
||||
export const INCLUDE_ENCRYPTED_REASONING = ["reasoning.encrypted_content"] as const
|
||||
|
||||
export interface Target {
|
||||
readonly npm?: string
|
||||
readonly apiID: string
|
||||
readonly modelID: string
|
||||
readonly providerID: string
|
||||
}
|
||||
|
||||
export function fromOptions(
|
||||
target: Target,
|
||||
options: ReadonlyArray<{ readonly type: string; readonly values?: ReadonlyArray<string | null> }> | undefined,
|
||||
): Record<string, Record<string, unknown>> | undefined {
|
||||
const efforts = [
|
||||
...new Set(
|
||||
(options ?? [])
|
||||
.flatMap((option) => (option.type === "effort" ? (option.values ?? []) : []))
|
||||
.filter((value): value is string => typeof value === "string"),
|
||||
),
|
||||
]
|
||||
if (efforts.length === 0) return undefined
|
||||
return effortVariants(target, efforts)
|
||||
}
|
||||
|
||||
export function anthropicOpus47OrLater(apiID: string) {
|
||||
// Matches "opus-4.7" (Anthropic/Bedrock/Vertex) and "claude-4.7-opus" (SAP AI Core inverted).
|
||||
// Greedy \d+ correctly extends to multi-digit majors (e.g. "claude-10.0-opus") for forward compatibility.
|
||||
const version = /opus-(\d+)[.-](\d+)(?:[.@-]|$)|claude-(\d+)[.-](\d+)-opus(?:[.@-]|$)/i.exec(apiID)
|
||||
if (!version) return false
|
||||
const major = Number(version[1] ?? version[3])
|
||||
const minor = Number(version[2] ?? version[4])
|
||||
return major > 4 || (major === 4 && minor >= 7)
|
||||
}
|
||||
|
||||
export function anthropicAdaptiveEfforts(apiID: string): string[] | null {
|
||||
if (anthropicOpus47OrLater(apiID) || apiID.includes("fable-5")) {
|
||||
return ["low", "medium", "high", "xhigh", "max"]
|
||||
}
|
||||
if (
|
||||
["opus-4-6", "opus-4.6", "4-6-opus", "4.6-opus", "sonnet-4-6", "sonnet-4.6", "4-6-sonnet", "4.6-sonnet"].some((v) =>
|
||||
apiID.includes(v),
|
||||
)
|
||||
) {
|
||||
return ["low", "medium", "high", "max"]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function anthropicOmitsThinking(apiID: string) {
|
||||
return anthropicOpus47OrLater(apiID) || apiID.includes("fable-5")
|
||||
}
|
||||
|
||||
// SAP's Zod schema drops unknown top-level keys; reasoning controls survive
|
||||
// only via `modelParams` (catchall), forwarded verbatim by the SAP SDKs.
|
||||
export function wrapInSapModelParams(
|
||||
variants: Record<string, Record<string, unknown>>,
|
||||
): Record<string, Record<string, unknown>> {
|
||||
return Object.fromEntries(Object.entries(variants).map(([k, v]) => [k, { modelParams: v }]))
|
||||
}
|
||||
|
||||
function copilotAnthropicEfforts(apiID: string, efforts: string[]) {
|
||||
// Efforts currently supported by copilot are: low, medium, high
|
||||
if (apiID.includes("opus-4.7")) return ["medium"]
|
||||
return efforts.filter((v) => v !== "max" && v !== "xhigh")
|
||||
}
|
||||
|
||||
function anthropicEffortVariants(target: Target, efforts: string[]): Record<string, Record<string, unknown>> {
|
||||
const filtered = target.providerID === "github-copilot" ? copilotAnthropicEfforts(target.apiID, efforts) : efforts
|
||||
const adaptive = anthropicAdaptiveEfforts(target.apiID) !== null
|
||||
return Object.fromEntries(
|
||||
filtered.map((effort) => [
|
||||
effort,
|
||||
adaptive
|
||||
? {
|
||||
thinking: {
|
||||
type: "adaptive",
|
||||
// Newer adaptive-only models default `display` to "omitted", which
|
||||
// returns empty thinking blocks. Force "summarized" so summaries
|
||||
// survive (4.6/Sonnet 4.6 already default to "summarized").
|
||||
...(anthropicOmitsThinking(target.apiID) ? { display: "summarized" } : {}),
|
||||
},
|
||||
effort,
|
||||
}
|
||||
: { effort },
|
||||
]),
|
||||
)
|
||||
}
|
||||
|
||||
function effortVariants(target: Target, efforts: string[]): Record<string, Record<string, unknown>> {
|
||||
const fromEffort = (encode: (effort: string) => Record<string, unknown>) =>
|
||||
Object.fromEntries(efforts.map((effort) => [effort, encode(effort)]))
|
||||
|
||||
switch (target.npm) {
|
||||
case "@openrouter/ai-sdk-provider":
|
||||
return fromEffort((effort) => ({ reasoning: { effort } }))
|
||||
|
||||
case "@ai-sdk/gateway":
|
||||
if (target.modelID.includes("anthropic")) return anthropicEffortVariants(target, efforts)
|
||||
if (target.modelID.includes("google"))
|
||||
return fromEffort((effort) => ({ includeThoughts: true, thinkingLevel: effort }))
|
||||
return fromEffort((effort) => ({ reasoningEffort: effort }))
|
||||
|
||||
case "@ai-sdk/github-copilot":
|
||||
// currently github copilot only returns thinking
|
||||
if (target.modelID.includes("gemini")) return {}
|
||||
if (target.modelID.includes("claude")) return fromEffort((effort) => ({ reasoningEffort: effort }))
|
||||
return fromEffort((effort) => ({
|
||||
reasoningEffort: effort,
|
||||
reasoningSummary: "auto",
|
||||
include: INCLUDE_ENCRYPTED_REASONING,
|
||||
}))
|
||||
|
||||
case "@ai-sdk/azure":
|
||||
case "@ai-sdk/amazon-bedrock/mantle":
|
||||
case "@ai-sdk/openai":
|
||||
return fromEffort((effort) => ({
|
||||
reasoningEffort: effort,
|
||||
reasoningSummary: "auto",
|
||||
include: INCLUDE_ENCRYPTED_REASONING,
|
||||
}))
|
||||
|
||||
case "@ai-sdk/anthropic":
|
||||
case "@ai-sdk/google-vertex/anthropic":
|
||||
return anthropicEffortVariants(target, efforts)
|
||||
|
||||
case "@ai-sdk/amazon-bedrock":
|
||||
if (anthropicAdaptiveEfforts(target.apiID)) {
|
||||
return fromEffort((effort) => ({
|
||||
reasoningConfig: {
|
||||
type: "adaptive",
|
||||
maxReasoningEffort: effort,
|
||||
...(anthropicOmitsThinking(target.apiID) ? { display: "summarized" } : {}),
|
||||
},
|
||||
}))
|
||||
}
|
||||
return fromEffort((effort) => ({
|
||||
reasoningConfig: {
|
||||
type: "enabled",
|
||||
maxReasoningEffort: effort,
|
||||
},
|
||||
}))
|
||||
|
||||
case "@ai-sdk/google-vertex":
|
||||
case "@ai-sdk/google":
|
||||
return fromEffort((effort) => ({ thinkingConfig: { includeThoughts: true, thinkingLevel: effort } }))
|
||||
|
||||
case "@jerome-benoit/sap-ai-provider-v2": {
|
||||
if (target.modelID.toLowerCase().includes("anthropic")) {
|
||||
const adaptive = anthropicAdaptiveEfforts(target.apiID) !== null
|
||||
// Bedrock-flavored Anthropic splits `effort` out into `output_config` (vs
|
||||
// Anthropic native which inlines it).
|
||||
return wrapInSapModelParams(
|
||||
fromEffort((effort) =>
|
||||
adaptive
|
||||
? {
|
||||
thinking: {
|
||||
type: "adaptive",
|
||||
...(anthropicOmitsThinking(target.apiID) ? { display: "summarized" } : {}),
|
||||
},
|
||||
output_config: { effort },
|
||||
}
|
||||
: { output_config: { effort } },
|
||||
),
|
||||
)
|
||||
}
|
||||
return wrapInSapModelParams(fromEffort((effort) => ({ reasoning_effort: effort })))
|
||||
}
|
||||
}
|
||||
|
||||
// OpenAI-compatible `reasoning_effort` is the dominant convention; it covers
|
||||
// @ai-sdk/openai-compatible, cerebras, togetherai, xai, deepinfra, venice,
|
||||
// mistral, groq, ai-gateway-provider, and unknown future packages.
|
||||
return fromEffort((effort) => ({ reasoningEffort: effort }))
|
||||
}
|
||||
|
|
@ -12,6 +12,28 @@ export const Model = Schema.Struct({
|
|||
release_date: Schema.optional(Schema.String),
|
||||
attachment: Schema.optional(Schema.Boolean),
|
||||
reasoning: Schema.optional(Schema.Boolean),
|
||||
reasoning_options: Schema.optional(
|
||||
// Mirrors the models.dev reasoning_options union; effort values stay open
|
||||
// strings so new tiers don't break configs.
|
||||
Schema.mutable(
|
||||
Schema.Array(
|
||||
Schema.Union([
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("toggle"),
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("effort"),
|
||||
values: Schema.mutable(Schema.Array(Schema.NullOr(Schema.String))),
|
||||
}),
|
||||
Schema.Struct({
|
||||
type: Schema.Literal("budget_tokens"),
|
||||
min: Schema.optional(Schema.Finite),
|
||||
max: Schema.optional(Schema.Finite),
|
||||
}),
|
||||
]),
|
||||
),
|
||||
),
|
||||
).annotate({ description: "Reasoning controls this model supports; effort values drive reasoning variants" }),
|
||||
temperature: Schema.optional(Schema.Boolean),
|
||||
tool_call: Schema.optional(Schema.Boolean),
|
||||
interleaved: Schema.optional(
|
||||
|
|
|
|||
|
|
@ -568,7 +568,12 @@ describe("Config", () => {
|
|||
model: {
|
||||
request: {
|
||||
body: {
|
||||
output_config: { effort: "high", task_budget: 4096 },
|
||||
// `effort` stays aisdk-shaped: the catalog re-partitions it
|
||||
// into the semantic anthropic effort option, which lowers to
|
||||
// output_config.effort plus the effort beta header. taskBudget
|
||||
// has no semantic and lowers to a raw output_config overlay.
|
||||
effort: "high",
|
||||
output_config: { task_budget: 4096 },
|
||||
metadata: { user_id: "user-1" },
|
||||
},
|
||||
},
|
||||
|
|
|
|||
116
packages/core/test/models-dev-plugin.test.ts
Normal file
116
packages/core/test/models-dev-plugin.test.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
import { describe, expect } from "bun:test"
|
||||
import { Effect, Layer } from "effect"
|
||||
import { Catalog } from "@opencode-ai/core/catalog"
|
||||
import { EventV2 } from "@opencode-ai/core/event"
|
||||
import { Location } from "@opencode-ai/core/location"
|
||||
import { ModelV2 } from "@opencode-ai/core/model"
|
||||
import { ModelsDev } from "@opencode-ai/core/models-dev"
|
||||
import { ModelsDevPlugin } from "@opencode-ai/core/plugin/models-dev"
|
||||
import { ProviderV2 } from "@opencode-ai/core/provider"
|
||||
import { AbsolutePath } from "@opencode-ai/core/schema"
|
||||
import { location } from "./fixture/location"
|
||||
import { testEffect } from "./lib/effect"
|
||||
|
||||
const model = (input: Partial<ModelsDev.Model> & { id: string }): ModelsDev.Model => ({
|
||||
name: input.id,
|
||||
release_date: "2026-01-01",
|
||||
attachment: false,
|
||||
reasoning: true,
|
||||
temperature: true,
|
||||
tool_call: true,
|
||||
limit: { context: 200_000, output: 64_000 },
|
||||
...input,
|
||||
})
|
||||
|
||||
const fixture: Record<string, ModelsDev.Provider> = {
|
||||
anthropic: {
|
||||
id: "anthropic",
|
||||
name: "Anthropic",
|
||||
env: [],
|
||||
npm: "@ai-sdk/anthropic",
|
||||
models: {
|
||||
"claude-sonnet-4-6": model({
|
||||
id: "claude-sonnet-4-6",
|
||||
reasoning_options: [
|
||||
{ type: "effort", values: ["low", "medium", "high", "max"] },
|
||||
{ type: "budget_tokens", min: 1024 },
|
||||
],
|
||||
}),
|
||||
},
|
||||
},
|
||||
compat: {
|
||||
id: "compat",
|
||||
name: "Compat",
|
||||
env: [],
|
||||
npm: "@ai-sdk/openai-compatible",
|
||||
models: {
|
||||
"deepseek-v4": model({
|
||||
id: "deepseek-v4",
|
||||
reasoning_options: [{ type: "toggle" }, { type: "effort", values: [null, "high", "max"] }],
|
||||
experimental: {
|
||||
modes: {
|
||||
high: { provider: { body: { reasoning_effort: "high", custom: true } } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const locationLayer = Layer.succeed(
|
||||
Location.Service,
|
||||
Location.Service.of(location({ directory: AbsolutePath.make("test") })),
|
||||
)
|
||||
const modelsDevLayer = Layer.succeed(
|
||||
ModelsDev.Service,
|
||||
ModelsDev.Service.of({
|
||||
get: () => Effect.succeed(fixture),
|
||||
refresh: () => Effect.void,
|
||||
}),
|
||||
)
|
||||
const it = testEffect(
|
||||
Layer.mergeAll(modelsDevLayer, Catalog.locationLayer).pipe(
|
||||
Layer.provideMerge(EventV2.defaultLayer),
|
||||
Layer.provideMerge(locationLayer),
|
||||
),
|
||||
)
|
||||
|
||||
describe("ModelsDevPlugin reasoning_options", () => {
|
||||
it.effect("generates anthropic effort variants as semantic thinking + effort options", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* ModelsDevPlugin.effect
|
||||
const catalog = yield* Catalog.Service
|
||||
const info = yield* catalog.model.get(ProviderV2.ID.make("anthropic"), ModelV2.ID.make("claude-sonnet-4-6"))
|
||||
expect(info.variants.map((variant) => variant.id)).toEqual(
|
||||
["low", "medium", "high", "max"].map((id) => ModelV2.VariantID.make(id)),
|
||||
)
|
||||
expect(info.variants[2]).toMatchObject({
|
||||
id: "high",
|
||||
headers: {},
|
||||
body: {},
|
||||
options: { thinking: { type: "adaptive" }, effort: "high" },
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
it.effect("merges effort variants after curated experimental modes, skipping null values and collisions", () =>
|
||||
Effect.gen(function* () {
|
||||
yield* ModelsDevPlugin.effect
|
||||
const catalog = yield* Catalog.Service
|
||||
const info = yield* catalog.model.get(ProviderV2.ID.make("compat"), ModelV2.ID.make("deepseek-v4"))
|
||||
expect(info.variants.map((variant) => variant.id)).toEqual(["high", "max"].map((id) => ModelV2.VariantID.make(id)))
|
||||
// curated mode wins the "high" id; its body keys survive partitioning
|
||||
expect(info.variants[0]).toMatchObject({
|
||||
id: "high",
|
||||
body: { custom: true },
|
||||
options: { reasoningEffort: "high" },
|
||||
})
|
||||
// data-driven effort variant for "max" uses the openai-compatible encoding
|
||||
expect(info.variants[1]).toMatchObject({
|
||||
id: "max",
|
||||
body: {},
|
||||
options: { reasoningEffort: "max" },
|
||||
})
|
||||
}),
|
||||
)
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue