diff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.py index 24b6dfb36d..6a0d49b47d 100644 --- a/studio/backend/routes/chat_history.py +++ b/studio/backend/routes/chat_history.py @@ -160,11 +160,26 @@ class ChatInferenceSettings(BaseModel): fastMode: Optional[bool] = None +class ChatPresetLoadConfig(BaseModel): + model_config = ConfigDict(extra = "forbid") + + customContextLength: Optional[int] = Field(default = None, gt = 0) + maxSeqLength: Optional[float] = None + kvCacheDtype: Optional[str] = None + speculativeType: Optional[str] = None + specDraftNMax: Optional[int] = Field(default = None, ge = 1, le = 16) + tensorParallel: Optional[bool] = None + gpuMemoryMode: Optional[Literal["manual"]] = None + gpuLayers: Optional[int] = None + nCpuMoe: Optional[int] = Field(default = None, ge = 0) + + class ChatPreset(BaseModel): model_config = ConfigDict(extra = "forbid") name: str params: ChatInferenceSettings + loadConfig: Optional[ChatPresetLoadConfig] = None class ChatSettingsPayload(BaseModel): diff --git a/studio/backend/tests/test_chat_history_routes.py b/studio/backend/tests/test_chat_history_routes.py index a60ac700bf..896bf1a6cd 100644 --- a/studio/backend/tests/test_chat_history_routes.py +++ b/studio/backend/tests/test_chat_history_routes.py @@ -91,6 +91,28 @@ def test_chat_settings_payload_accepts_fast_mode_presets(): assert dumped["customPresets"][0]["params"]["fastMode"] is True +def test_chat_settings_payload_accepts_preset_load_config(): + payload = chat_history.ChatSettingsPayload.model_validate( + { + "customPresets": [ + { + "name": "GGUF preset", + "params": {"temperature": 0.7, "maxTokens": 512}, + "loadConfig": { + "customContextLength": 256, + "kvCacheDtype": "q8_0", + "tensorParallel": False, + }, + }, + ], + } + ) + + dumped = payload.model_dump(exclude_unset = True) + assert dumped["customPresets"][0]["loadConfig"]["customContextLength"] == 256 + assert dumped["customPresets"][0]["loadConfig"]["kvCacheDtype"] == "q8_0" + + def test_chat_settings_payload_accepts_nudge_tool_calls(): # extra="forbid" 400s PUT /api/chat/settings on unknown keys, so the # frontend's persisted nudgeToolCalls needs a payload field (like diff --git a/studio/frontend/src/features/chat/api/chat-settings-api.ts b/studio/frontend/src/features/chat/api/chat-settings-api.ts index 1e00357ea4..ab13130525 100644 --- a/studio/frontend/src/features/chat/api/chat-settings-api.ts +++ b/studio/frontend/src/features/chat/api/chat-settings-api.ts @@ -13,6 +13,7 @@ export type PersistedInferenceParams = Partial< export interface PersistedChatPreset { name: string; params: PersistedInferenceParams; + loadConfig?: Record; } export interface PersistedChatSettings { diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index 672ab2b2d3..c3a59e9860 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -75,6 +75,12 @@ import { isSamePresetConfig, toPresetParams, } from "./presets/preset-policy"; +import { + applyPresetLoadConfig, + capturePresetLoadConfig, + formatPresetLoadConfigSummary, + isSamePresetLoadConfig, +} from "./presets/preset-load-config"; import { type ProviderCapabilities, getExternalMaxOutputTokens, @@ -385,6 +391,12 @@ export function ChatSettingsPanel({ (s) => s.ggufMaxContextLength, ); const customContextLength = useChatRuntimeStore((s) => s.customContextLength); + const kvCacheDtype = useChatRuntimeStore((s) => s.kvCacheDtype); + const gpuMemoryMode = useChatRuntimeStore((s) => s.gpuMemoryMode); + const gpuLayers = useChatRuntimeStore((s) => s.gpuLayers); + const nCpuMoe = useChatRuntimeStore((s) => s.nCpuMoe); + const tensorParallel = useChatRuntimeStore((s) => s.tensorParallel); + const specDraftNMax = useChatRuntimeStore((s) => s.specDraftNMax); const speculativeType = useChatRuntimeStore((s) => s.speculativeType); const specFallbackReason = useChatRuntimeStore((s) => s.specFallbackReason); const mtpUpdatable = @@ -469,11 +481,50 @@ export function ChatSettingsPanel({ if (activePresetDefinition == null) { return false; } - if (activePresetDefinition.name === "Default") { - return activePresetSource === "modified"; - } - return !isSamePresetConfig(activePresetDefinition.params, params); - }, [activePresetDefinition, activePresetSource, params]); + const samplingChanged = + activePresetDefinition.name === "Default" + ? activePresetSource === "modified" + : !isSamePresetConfig(activePresetDefinition.params, params); + const currentLoadConfig = capturePresetLoadConfig(); + const loadChanged = !isSamePresetLoadConfig( + activePresetDefinition.loadConfig, + currentLoadConfig, + ); + return samplingChanged || loadChanged; + }, [ + activePresetDefinition, + activePresetSource, + params, + customContextLength, + ggufContextLength, + kvCacheDtype, + gpuMemoryMode, + gpuLayers, + nCpuMoe, + tensorParallel, + speculativeType, + specDraftNMax, + params.maxSeqLength, + ]); + const activePresetLoadSummary = useMemo( + () => formatPresetLoadConfigSummary(activePresetDefinition?.loadConfig), + [activePresetDefinition], + ); + const currentLoadSummary = useMemo( + () => formatPresetLoadConfigSummary(capturePresetLoadConfig()), + [ + customContextLength, + ggufContextLength, + kvCacheDtype, + gpuMemoryMode, + gpuLayers, + nCpuMoe, + tensorParallel, + speculativeType, + specDraftNMax, + params.maxSeqLength, + ], + ); const presetSaveState = useMemo( () => getPresetSaveState({ @@ -549,8 +600,14 @@ export function ChatSettingsPanel({ onParamsChange({ ...applyPresetParams(params, p.params), }); + if (p.loadConfig) { + applyPresetLoadConfig(p.loadConfig); + } setActivePreset(name); setActivePresetSource(getPresetSource(name)); + if (p.loadConfig && params.checkpoint) { + toast.info("Reload the model to apply load settings from this preset."); + } } } @@ -571,9 +628,14 @@ export function ChatSettingsPanel({ ? getBuiltinVariantName(trimmed, usedNames) : trimmed; const next = customPresets.filter((p) => p.name !== saveName); + const loadConfig = capturePresetLoadConfig(); const merged = [ ...next, - { name: saveName, params: toPresetParams(params) }, + { + name: saveName, + params: toPresetParams(params), + ...(loadConfig ? { loadConfig } : {}), + }, ]; setCustomPresets(merged); setActivePreset(saveName); @@ -598,8 +660,11 @@ export function ChatSettingsPanel({ if (activePreset === name) { if (fallbackPreset) { onParamsChange({ - ...applyPresetParams(params, fallbackPreset.params), + ... applyPresetParams(params, fallbackPreset.params), }); + if (fallbackPreset.loadConfig) { + applyPresetLoadConfig(fallbackPreset.loadConfig); + } setActivePreset(fallbackPreset.name); setActivePresetSource("builtin-default"); } @@ -901,6 +966,23 @@ export function ChatSettingsPanel({ Delete +

+ Saving a preset also stores current load settings (context length, + KV cache dtype, speculative decoding, GPU layers). + {currentLoadSummary ? ( + <> + {" "} + Active now: {currentLoadSummary}. + + ) : null} + {activePresetLoadSummary && + activePresetLoadSummary !== currentLoadSummary ? ( + <> + {" "} + Saved in preset: {activePresetLoadSummary}. + + ) : null} +

diff --git a/studio/frontend/src/features/chat/presets/preset-load-config.ts b/studio/frontend/src/features/chat/presets/preset-load-config.ts new file mode 100644 index 0000000000..1083655cf2 --- /dev/null +++ b/studio/frontend/src/features/chat/presets/preset-load-config.ts @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + applyPerModelConfigToRuntime, + currentRuntimePerModelConfig, + perModelConfigsEqual, +} from "@/features/model-picker"; +import { + CONTEXT_LENGTH_MIN, + DEFAULT_PER_MODEL_CONFIG, + DEFAULT_MAX_SEQ_LENGTH, + KV_CACHE_DTYPES, + MTP_SPECULATIVE_TYPES, + SPECULATIVE_TYPES, + normalizeMaxSeqLength, + type PerModelConfig, +} from "@/features/model-picker/model-config/per-model-config"; +import { + GPU_LAYERS_AUTO, + useChatRuntimeStore, + normalizeSpeculativeType, +} from "../stores/chat-runtime-store"; + +/** Load/runtime knobs saved in a chat preset (excludes per-model-only blobs). */ +export type PresetLoadConfig = Pick< + PerModelConfig, + | "customContextLength" + | "maxSeqLength" + | "kvCacheDtype" + | "speculativeType" + | "specDraftNMax" + | "tensorParallel" + | "gpuMemoryMode" + | "gpuLayers" + | "nCpuMoe" +>; + +const VALID_KV_CACHE_DTYPES = new Set(KV_CACHE_DTYPES); +const VALID_SPECULATIVE_TYPES = new Set(SPECULATIVE_TYPES); + +export const EMPTY_PRESET_LOAD_CONFIG: PresetLoadConfig = { + customContextLength: null, + maxSeqLength: null, + kvCacheDtype: null, + speculativeType: null, + specDraftNMax: null, + tensorParallel: false, +}; + +function toComparablePerModelConfig( + config: PresetLoadConfig, +): PerModelConfig { + return { + ...DEFAULT_PER_MODEL_CONFIG, + ...config, + chatTemplateOverride: null, + selectedGpuIds: null, + }; +} + +export function normalizePresetLoadConfig( + raw: unknown, +): PresetLoadConfig | undefined { + if (raw == null || typeof raw !== "object" || Array.isArray(raw)) { + return undefined; + } + const partial = raw as Record; + const rawSpecType = + typeof partial.speculativeType === "string" + ? normalizeSpeculativeType(partial.speculativeType) + : null; + const speculativeType = rawSpecType ?? null; + const specDraftNMax = + speculativeType != null && + MTP_SPECULATIVE_TYPES.has(speculativeType) && + typeof partial.specDraftNMax === "number" && + Number.isFinite(partial.specDraftNMax) + ? Math.max(1, Math.min(16, Math.round(partial.specDraftNMax))) + : null; + const gpuMemoryMode = + partial.gpuMemoryMode === "manual" ? ("manual" as const) : undefined; + let gpuLayers: number | undefined; + if (typeof partial.gpuLayers === "number" && Number.isFinite(partial.gpuLayers)) { + gpuLayers = partial.gpuLayers < 0 ? GPU_LAYERS_AUTO : Math.floor(partial.gpuLayers); + } + let nCpuMoe: number | undefined; + if (typeof partial.nCpuMoe === "number" && Number.isFinite(partial.nCpuMoe)) { + nCpuMoe = Math.max(0, Math.floor(partial.nCpuMoe)); + } + + const normalized: PresetLoadConfig = { + customContextLength: + typeof partial.customContextLength === "number" && + Number.isFinite(partial.customContextLength) && + partial.customContextLength > 0 + ? Math.max(CONTEXT_LENGTH_MIN, Math.floor(partial.customContextLength)) + : null, + maxSeqLength: normalizeMaxSeqLength(partial.maxSeqLength as number | null), + kvCacheDtype: + typeof partial.kvCacheDtype === "string" && + VALID_KV_CACHE_DTYPES.has(partial.kvCacheDtype) + ? partial.kvCacheDtype + : null, + speculativeType: + speculativeType && VALID_SPECULATIVE_TYPES.has(speculativeType) + ? speculativeType + : null, + specDraftNMax, + tensorParallel: + typeof partial.tensorParallel === "boolean" + ? partial.tensorParallel + : false, + ...(gpuMemoryMode ? { gpuMemoryMode } : {}), + ...(gpuLayers !== undefined ? { gpuLayers } : {}), + ...(nCpuMoe !== undefined ? { nCpuMoe } : {}), + }; + + return hasPresetLoadConfig(normalized) ? normalized : undefined; +} + +export function hasPresetLoadConfig( + config?: PresetLoadConfig | null, +): boolean { + return !isSamePresetLoadConfig(config, EMPTY_PRESET_LOAD_CONFIG); +} + +export function isSamePresetLoadConfig( + a?: PresetLoadConfig | null, + b?: PresetLoadConfig | null, +): boolean { + return perModelConfigsEqual( + toComparablePerModelConfig({ ...EMPTY_PRESET_LOAD_CONFIG, ...a }), + toComparablePerModelConfig({ ...EMPTY_PRESET_LOAD_CONFIG, ...b }), + ); +} + +export function capturePresetLoadConfig(): PresetLoadConfig | undefined { + const snapshot = currentRuntimePerModelConfig({ includeMaxSeqLength: true }); + const store = useChatRuntimeStore.getState(); + const isGguf = + store.activeGgufVariant != null || + store.ggufContextLength != null || + (store.params.checkpoint?.toLowerCase().endsWith(".gguf") ?? false); + const effectiveContextLength = + snapshot.customContextLength ?? + (isGguf ? store.ggufContextLength : null); + const captured: PresetLoadConfig = { + customContextLength: effectiveContextLength ?? null, + maxSeqLength: normalizeMaxSeqLength(snapshot.maxSeqLength), + kvCacheDtype: snapshot.kvCacheDtype ?? null, + speculativeType: normalizeSpeculativeType(snapshot.speculativeType), + specDraftNMax: snapshot.specDraftNMax ?? null, + tensorParallel: snapshot.tensorParallel ?? false, + ...(snapshot.gpuMemoryMode === "manual" + ? { gpuMemoryMode: "manual" as const } + : {}), + ...(snapshot.gpuLayers != null && snapshot.gpuLayers >= 0 + ? { gpuLayers: snapshot.gpuLayers } + : snapshot.gpuMemoryMode === "manual" + ? { gpuLayers: GPU_LAYERS_AUTO } + : {}), + ...(snapshot.nCpuMoe != null && snapshot.nCpuMoe > 0 + ? { nCpuMoe: snapshot.nCpuMoe } + : {}), + }; + return hasPresetLoadConfig(coalesceDefaultLoadKnobs(captured)) + ? coalesceDefaultLoadKnobs(captured) + : undefined; +} + +function coalesceDefaultLoadKnobs( + captured: PresetLoadConfig, +): PresetLoadConfig { + const result: PresetLoadConfig = { ...captured }; + if (normalizeMaxSeqLength(result.maxSeqLength) === DEFAULT_MAX_SEQ_LENGTH) { + result.maxSeqLength = null; + } + const speculativeType = normalizeSpeculativeType(result.speculativeType); + if (speculativeType == null || speculativeType === "auto") { + result.speculativeType = null; + } + if ( + (result.gpuLayers == null || result.gpuLayers < 0) && + result.gpuMemoryMode !== "manual" + ) { + delete result.gpuLayers; + } + if ((result.nCpuMoe ?? 0) === 0) { + delete result.nCpuMoe; + } + return result; +} + +export function applyPresetLoadConfig( + config?: PresetLoadConfig | null, +): void { + if (config == null) { + return; + } + const store = useChatRuntimeStore.getState(); + applyPerModelConfigToRuntime({ + ...DEFAULT_PER_MODEL_CONFIG, + maxSeqLength: normalizeMaxSeqLength(config.maxSeqLength) ?? DEFAULT_MAX_SEQ_LENGTH, + customContextLength: config.customContextLength ?? null, + kvCacheDtype: config.kvCacheDtype ?? null, + speculativeType: config.speculativeType ?? null, + specDraftNMax: config.specDraftNMax ?? null, + tensorParallel: config.tensorParallel ?? false, + chatTemplateOverride: null, + gpuMemoryMode: config.gpuMemoryMode, + gpuLayers: config.gpuLayers, + nCpuMoe: config.nCpuMoe, + selectedGpuIds: store.selectedGpuIds, + }); +} + +export function formatPresetLoadConfigSummary( + config?: PresetLoadConfig | null, +): string | null { + if (!config || !hasPresetLoadConfig(config)) { + return null; + } + const parts: string[] = []; + if (config.customContextLength != null) { + parts.push(`Ctx ${config.customContextLength.toLocaleString()}`); + } + if (config.kvCacheDtype) { + parts.push(`KV ${config.kvCacheDtype}`); + } + if (config.speculativeType && config.speculativeType !== "auto") { + parts.push(`Spec ${config.speculativeType}`); + } + if (config.gpuMemoryMode === "manual") { + parts.push("GPU manual"); + } + if (config.gpuLayers != null && config.gpuLayers >= 0) { + parts.push(`${config.gpuLayers} layers`); + } + if (config.tensorParallel) { + parts.push("TP"); + } + return parts.length > 0 ? parts.join(" ยท ") : null; +} diff --git a/studio/frontend/src/features/chat/presets/preset-policy.ts b/studio/frontend/src/features/chat/presets/preset-policy.ts index 23d79a35e1..61c94707b4 100644 --- a/studio/frontend/src/features/chat/presets/preset-policy.ts +++ b/studio/frontend/src/features/chat/presets/preset-policy.ts @@ -5,12 +5,15 @@ import { DEFAULT_INFERENCE_PARAMS, type InferenceParams, } from "../types/runtime"; +import type { PresetLoadConfig } from "./preset-load-config"; export const defaultInferenceParams = DEFAULT_INFERENCE_PARAMS; export interface Preset { name: string; params: InferenceParams; + /** Optional GGUF/load knobs captured with the preset. */ + loadConfig?: PresetLoadConfig; } export type PresetOwnedParams = Pick< @@ -85,6 +88,7 @@ export function normalizeCustomPresets(presets: Preset[]): Preset[] { return { name, params: preset.params, + ...(preset.loadConfig ? { loadConfig: preset.loadConfig } : {}), }; }) .filter((preset): preset is Preset => preset !== null); diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 15fa962dff..42359b8f7f 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -11,6 +11,7 @@ import { type Preset, getPresetSource, } from "../presets/preset-policy"; +import { normalizePresetLoadConfig } from "../presets/preset-load-config"; import { getExternalMaxOutputTokens } from "../provider-capabilities"; import { type ChatLoraSummary, @@ -1159,13 +1160,17 @@ function getHydratedCustomPresets( state: ChatRuntimeStore, ): Preset[] { return ( - settings.customPresets?.map((preset) => ({ - name: preset.name, - params: { - ...DEFAULT_INFERENCE_PARAMS, - ...preset.params, - }, - })) ?? state.customPresets + settings.customPresets?.map((preset) => { + const loadConfig = normalizePresetLoadConfig(preset.loadConfig); + return { + name: preset.name, + params: { + ...DEFAULT_INFERENCE_PARAMS, + ...preset.params, + }, + ...(loadConfig ? { loadConfig } : {}), + }; + }) ?? state.customPresets ); } diff --git a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts index 4899cb9c83..48acfdee40 100644 --- a/studio/frontend/src/features/chat/utils/chat-settings-storage.ts +++ b/studio/frontend/src/features/chat/utils/chat-settings-storage.ts @@ -8,6 +8,7 @@ import { type PersistedChatSettings, type PersistedInferenceParams, } from "../api/chat-settings-api"; +import { normalizePresetLoadConfig } from "../presets/preset-load-config"; import { BUILTIN_PRESETS, defaultInferenceParams, @@ -152,6 +153,7 @@ function sanitizeInferenceParams( } function toFullPreset(preset: PersistedChatPreset): Preset { + const loadConfig = normalizePresetLoadConfig(preset.loadConfig); return { name: preset.name, params: { @@ -159,6 +161,7 @@ function toFullPreset(preset: PersistedChatPreset): Preset { ...preset.params, checkpoint: defaultInferenceParams.checkpoint, }, + ...(loadConfig ? { loadConfig } : {}), }; } @@ -174,7 +177,12 @@ function sanitizeCustomPresets( const name = item.name.trim(); if (!name) return null; const params = sanitizeInferenceParams(item.params); - return { name, params: params ?? {} }; + const loadConfig = normalizePresetLoadConfig(item.loadConfig); + return { + name, + params: params ?? {}, + ...(loadConfig ? { loadConfig } : {}), + }; }) .filter((preset): preset is PersistedChatPreset => preset !== null); @@ -183,6 +191,7 @@ function sanitizeCustomPresets( (preset, index) => ({ name: preset.name, params: presets[index]?.params ?? {}, + ...(preset.loadConfig ? { loadConfig: preset.loadConfig } : {}), }), ); } diff --git a/tests/studio/test_chat_preset_load_config.py b/tests/studio/test_chat_preset_load_config.py new file mode 100644 index 0000000000..1588c7d96d --- /dev/null +++ b/tests/studio/test_chat_preset_load_config.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Contract coverage for preset load settings (#7347).""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def _read(relative: str) -> str: + path = ROOT / relative + if not path.exists(): + path = ROOT / "unsloth_repo" / relative + return path.read_text(encoding = "utf-8") + + +def test_preset_interface_includes_load_config(): + policy = _read("studio/frontend/src/features/chat/presets/preset-policy.ts") + assert "loadConfig?: PresetLoadConfig" in policy + + +def test_preset_save_captures_load_config(): + sheet = _read("studio/frontend/src/features/chat/chat-settings-sheet.tsx") + assert "capturePresetLoadConfig()" in sheet + assert "applyPresetLoadConfig" in sheet + + +def test_preset_apply_restores_load_config(): + sheet = _read("studio/frontend/src/features/chat/chat-settings-sheet.tsx") + assert "if (p.loadConfig)" in sheet + assert "applyPresetLoadConfig(p.loadConfig)" in sheet + + +def test_persisted_preset_serializes_load_config(): + storage = _read("studio/frontend/src/features/chat/utils/chat-settings-storage.ts") + assert "normalizePresetLoadConfig(item.loadConfig)" in storage + api = _read("studio/frontend/src/features/chat/api/chat-settings-api.ts") + assert "loadConfig?: Record" in api + + +def test_capture_reads_gguf_loaded_context(): + source = _read("studio/frontend/src/features/chat/presets/preset-load-config.ts") + assert "store.ggufContextLength" in source + assert "effectiveContextLength" in source + + +def test_apply_skips_missing_load_config(): + source = _read("studio/frontend/src/features/chat/presets/preset-load-config.ts") + assert "if (config == null)" in source + assert "selectedGpuIds: store.selectedGpuIds" in source + sheet = _read("studio/frontend/src/features/chat/chat-settings-sheet.tsx") + assert "if (p.loadConfig)" in sheet + + +def test_hydration_does_not_replay_preset_load_config(): + store = _read("studio/frontend/src/features/chat/stores/chat-runtime-store.ts") + assert "applyPresetLoadConfig(activeDefinition.loadConfig)" not in store + + +def test_capture_coalesces_default_load_knobs(): + source = _read("studio/frontend/src/features/chat/presets/preset-load-config.ts") + assert "coalesceDefaultLoadKnobs" in source + assert "DEFAULT_MAX_SEQ_LENGTH" in source + + +def test_backend_chat_preset_accepts_load_config(): + routes = _read("studio/backend/routes/chat_history.py") + assert "class ChatPresetLoadConfig" in routes + assert "loadConfig: Optional[ChatPresetLoadConfig]" in routes