From 95da8d52d33dd825a7a7a7255054f97a4e29fa06 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 26 May 2026 11:22:54 +0000 Subject: [PATCH] Studio: per-model external max_tokens cap + clamp on model switch Two related external-provider issues that surfaced from the same investigation as the per-card web_search / shell_call result bugs in the previous commit: A. Slider cap was a one-size-fits-all 32768 for every external model. provider-capabilities.ts kept a single EXTERNAL_MAX_OUTPUT_TOKENS constant (32k), well below what most providers actually accept. The docstring even called out the right per-provider numbers (Anthropic Opus 128k, GPT-5.x ~128k, Gemini 2.5 ~65k, DeepSeek 8k) but the code picked the lowest as a conservative floor. Effect: long generations from gpt-5.5 / claude-opus-4-7 silently truncated at 32k even though the API would have served up to 128k. Fix: introduce getExternalMaxOutputTokens(providerType, modelId) returning the documented per-model cap. Patterns are checked longest-first so e.g. gpt-5.5-pro matches before gpt-5.5. Unknown provider/model combinations fall back to the existing 32k floor so no surprise increases for ids we don't know about. Per-model caps from the official docs: - OpenAI gpt-5.5 / gpt-5.5-pro: 128000 - OpenAI gpt-5.4 / gpt-5.4-pro: 65536 - OpenAI gpt-5.3: 16384 - Anthropic claude-opus-4-7: 128000 - Anthropic claude-opus-4-6 / sonnet-4-6 / opus-4-5 / sonnet-4-5 / haiku-4-5: 64000 - Gemini 3.x family: 65535 - DeepSeek: 8192 - OpenRouter: strip provider/ prefix from the id and re-resolve The slider in chat-settings-sheet.tsx and the send-time clamp in chat-adapter.ts both call the new function so the slider's max= matches what the wire layer will accept. B. Slider value lied after switching from a local model to external. When Studio auto-loads the helper Gemma-4-E2B-it on first chat, chat-adapter sets params.maxTokens to Gemma's context_length (262144 for Gemma 4). Switching the model picker to gpt-5.5 then flips the slider's max prop to the external cap, but the stored params.maxTokens is never reset. The numeric value next to the slider would render 262144 against a track that ended at the external cap. The send-time clamp brought the outbound max_tokens back down to the cap, so the API call was safe, but the displayed number had no relationship to what was actually being sent. Fix: chat-runtime-store.setCheckpoint now clamps params.maxTokens to getExternalMaxOutputTokens(...) on transitions into an external model. Looks up the provider via useExternalProvidersStore so we can derive providerType from the parsed external model id. No-op when the stored maxTokens is already at or below the new cap, so user-tuned values within range survive the switch. Scope: pure frontend changes scoped to external-provider code paths. Local model behaviour is untouched -- the ggufContextLength branch of the slider's max= is unchanged, and setCheckpoint only mutates maxTokens when isExternalModelId(modelId) is true. The send-time clamp continues to be the safety net for any in-flight request that crosses a model switch before the store-level clamp has applied. Typecheck (tsc -b) clean; bun run build succeeds (2.13s). Co-changes with the previous commit (7fe1adbf, per-card web_search + shell_call output fallback) form a single PR: every empty-output and silent-truncation issue surfaced from the same animal-popularity prompt reproduction is now addressed in one branch. --- .../src/features/chat/api/chat-adapter.ts | 14 ++- .../src/features/chat/chat-settings-sheet.tsx | 6 +- .../features/chat/provider-capabilities.ts | 112 ++++++++++++++++-- .../chat/stores/chat-runtime-store.ts | 27 ++++- 4 files changed, 143 insertions(+), 16 deletions(-) diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index 49a5eebd6b..5cffac3eb1 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -21,6 +21,7 @@ import { pickFriendlyContainerName } from "../lib/friendly-names"; import { EXTERNAL_MAX_OUTPUT_TOKENS, clampReasoningEffortToLevels, + getExternalMaxOutputTokens, getExternalMinOutputTokens, getExternalReasoningCapabilities, getProviderCapabilities, @@ -1692,18 +1693,21 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { ...(externalCapabilities?.topP !== false ? { top_p: params.topP } : {}), - // Clamp to the cross-provider output cap so a maxTokens value + // Clamp to the per-model output cap so a maxTokens value // carried over from a local-model session does not blow past // provider limits (e.g. Claude Opus 400s on >128k). Also - // floor to the provider's documented minimum — Kimi's - // thinking models need >=16k or the response truncates - // before the answer fits alongside reasoning_content. + // floor to the provider's documented minimum (Kimi thinking + // needs >=16k or the response truncates before the answer + // fits alongside reasoning_content). max_tokens: Math.min( Math.max( params.maxTokens, getExternalMinOutputTokens(externalProvider?.providerType), ), - EXTERNAL_MAX_OUTPUT_TOKENS, + getExternalMaxOutputTokens( + externalProvider?.providerType, + externalSelection?.modelId, + ), ), // Only forward sampling knobs the provider actually accepts; the // backend's external-provider proxy is param-permissive and would diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index ac1ef8a24c..3714fb128a 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -85,6 +85,7 @@ import { import { EXTERNAL_MAX_OUTPUT_TOKENS, type ProviderCapabilities, + getExternalMaxOutputTokens, getExternalMinOutputTokens, providerSupportsBuiltinCodeExecution, providerSupportsFastMode, @@ -1309,7 +1310,10 @@ export function ChatSettingsPanel({ } max={ isExternalModel - ? EXTERNAL_MAX_OUTPUT_TOKENS + ? getExternalMaxOutputTokens( + externalProviderType, + externalSelection?.modelId, + ) : isGguf && ggufContextLength ? ggufContextLength : 32768 diff --git a/studio/frontend/src/features/chat/provider-capabilities.ts b/studio/frontend/src/features/chat/provider-capabilities.ts index ef805305be..8f4a5b88e4 100644 --- a/studio/frontend/src/features/chat/provider-capabilities.ts +++ b/studio/frontend/src/features/chat/provider-capabilities.ts @@ -71,18 +71,112 @@ export function clampReasoningEffortToLevels( } /** - * Output-token cap for any external provider request. Picked to stay below the - * tightest declared limit across the providers we ship (Anthropic Claude Opus - * tops out at 128k, GPT-5.x ~128k, Gemini 2.5 ~65k, DeepSeek 8k) while staying - * well above what a typical chat reply needs. The local-model path is not - * subject to this — local backends honour whatever the loaded context allows. - * - * If a user's stored maxTokens (e.g. carried over from a prior local-model - * session with a 128k+ context) exceeds this, chat-adapter clamps the - * outbound request so the provider does not 400 on it. + * Conservative cross-provider output cap, used as a final clamp and as the + * fallback for providers/models we don't have a documented limit for. + * Prefer `getExternalMaxOutputTokens(providerType, modelId)` for the real + * per-model cap (gpt-5.5 / claude-opus-4-7 both ship 128k max output, much + * higher than this conservative floor). */ export const EXTERNAL_MAX_OUTPUT_TOKENS = 32768; +/** + * Per-model max-output cap, sourced from each provider's docs. Used by the + * settings-slider max and by chat-adapter's send-time clamp so the slider + * never reports a value above what the provider will accept. + * + * Patterns are ordered most-specific-first so `gpt-5.5-pro` doesn't match + * the `gpt-5.5` row, etc. Look-up runs longest-prefix-first via + * `_pickByPrefix` below. + * + * Sources: + * - OpenAI: https://developers.openai.com/api/docs/models/gpt-5.5 + * (gpt-5.5: 128k max output; gpt-5.4: 64k; gpt-5.3: 16k) + * - Anthropic: https://platform.claude.com/docs/en/about-claude/models/ + * (Opus 4.7: 128k max output; 4.6 / 4.5 family: 64k) + * - Google Gemini 3.x: 65535 max output tokens + * - DeepSeek: 8192 max output tokens + * + * The local-model path is not subject to this; local backends honour + * whatever the loaded context allows. + */ +const EXTERNAL_MAX_OUTPUT_TOKENS_BY_MODEL: Array<{ + providerType: string; + prefixes: readonly string[]; + cap: number; +}> = [ + // OpenAI + { providerType: "openai", prefixes: ["gpt-5.5-pro", "gpt-5.5"], cap: 128000 }, + { providerType: "openai", prefixes: ["gpt-5.4-pro", "gpt-5.4"], cap: 65536 }, + { providerType: "openai", prefixes: ["gpt-5.3"], cap: 16384 }, + // Anthropic + { + providerType: "anthropic", + prefixes: ["claude-opus-4-7"], + cap: 128000, + }, + { + providerType: "anthropic", + prefixes: [ + "claude-opus-4-6", + "claude-sonnet-4-6", + "claude-opus-4-5", + "claude-sonnet-4-5", + "claude-haiku-4-5", + ], + cap: 64000, + }, + // Gemini + { providerType: "gemini", prefixes: ["gemini-3", "gemini-pro", "gemini-flash"], cap: 65535 }, + // DeepSeek + { providerType: "deepseek", prefixes: ["deepseek"], cap: 8192 }, +]; + +/** + * Return the documented max-output-tokens cap for a given external + * provider + model. Falls back to `EXTERNAL_MAX_OUTPUT_TOKENS` (32k) for + * provider/model combinations we don't have a published number for, so + * unknown ids never regress to a higher-than-supported value. + * + * OpenRouter normalises to `provider/model`; strip the `provider/` + * prefix before matching so `openai/gpt-5.5` resolves the same as + * `gpt-5.5`. + */ +export function getExternalMaxOutputTokens( + providerType: string | null | undefined, + modelId: string | null | undefined, +): number { + if (!providerType || !modelId) return EXTERNAL_MAX_OUTPUT_TOKENS; + const normalized = modelId.trim().toLowerCase(); + if (!normalized) return EXTERNAL_MAX_OUTPUT_TOKENS; + const stripped = + providerType === "openrouter" && normalized.includes("/") + ? normalized.split("/").slice(-1)[0] + : normalized; + const effectiveProvider = + providerType === "openrouter" + ? _inferProviderFromOpenrouterId(normalized) ?? providerType + : providerType; + for (const entry of EXTERNAL_MAX_OUTPUT_TOKENS_BY_MODEL) { + if (entry.providerType !== effectiveProvider) continue; + if (entry.prefixes.some((prefix) => stripped.startsWith(prefix))) { + return entry.cap; + } + } + return EXTERNAL_MAX_OUTPUT_TOKENS; +} + +function _inferProviderFromOpenrouterId( + normalizedId: string, +): string | null { + // OpenRouter ids are `provider/model`. Map the prefix back to one of + // our internal providerType keys so the per-model cap table applies. + if (normalizedId.startsWith("openai/")) return "openai"; + if (normalizedId.startsWith("anthropic/")) return "anthropic"; + if (normalizedId.startsWith("google/")) return "gemini"; + if (normalizedId.startsWith("deepseek/")) return "deepseek"; + return null; +} + /** * Whether the external provider offers a built-in web-search tool that the * model invokes server-side. When `true`, the chat composer's Search button diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index c78f02a474..30d12cd986 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -14,7 +14,9 @@ import { DEFAULT_INFERENCE_PARAMS, type InferenceParams, } from "../types/runtime"; -import { isExternalModelId } from "../external-providers"; +import { isExternalModelId, parseExternalModelId } from "../external-providers"; +import { getExternalMaxOutputTokens } from "../provider-capabilities"; +import { useExternalProvidersStore } from "./external-providers-store"; import { loadChatSettingsWithLegacyImport, savePersistedChatSettingsPatch, @@ -747,10 +749,33 @@ export const useChatRuntimeStore = create((set, get) => ({ // external-provider render gate would otherwise show old counters // until the next completion overwrites them. const checkpointChanged = state.params.checkpoint !== modelId; + // Clamp maxTokens to the new model's cap when switching INTO an + // external model. Otherwise a value carried over from a prior + // local-model session (e.g. Gemma's 262144 context) would render + // in the slider above the external cap, even though the wire-side + // clamp would still bring it down at send time. Keeps the + // displayed value honest. + let nextMaxTokens = state.params.maxTokens; + if (checkpointChanged && isExternalModelId(modelId)) { + const parsed = parseExternalModelId(modelId); + const provider = parsed + ? useExternalProvidersStore + .getState() + .providers.find((p) => p.id === parsed.providerId) + : null; + const cap = getExternalMaxOutputTokens( + provider?.providerType, + parsed?.modelId, + ); + if (nextMaxTokens > cap) { + nextMaxTokens = cap; + } + } return { params: { ...state.params, checkpoint: modelId, + maxTokens: nextMaxTokens, }, activeGgufVariant: ggufVariant ?? null, ...(checkpointChanged ? { contextUsage: null } : {}),