diff --git a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx index 30176a7d90..72686843f4 100644 --- a/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx +++ b/studio/frontend/src/features/hub/catalog/hub-model-settings-view.tsx @@ -10,9 +10,9 @@ import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker"; import type { PerModelConfig } from "@/features/model-picker"; +import { cn } from "@/lib/utils"; import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; -import { cn } from "@/lib/utils"; import { useEffect, useRef, useState } from "react"; export function HubModelSettingsView({ @@ -109,10 +109,10 @@ export function HubModelSettingsView({ />

- {/* Only a GGUF is mirrored to the server, because API auto-switch - indexes GGUFs only, so promising the API case for anything - else describes a load that cannot happen. */} - {target.isGguf + {/* Only what auto-switch can reach is mirrored to the server: it + indexes GGUFs and skips Ollama, so promising the API case for + anything else describes a load that cannot happen. */} + {(target.apiLoadable ?? target.isGguf) ? "Saved settings apply everywhere this model loads, including when an OpenAI-compatible API request asks for it." : "Saved settings apply everywhere Studio loads this model."}{" "} Turn on{" "} diff --git a/studio/frontend/src/features/hub/hub-page.tsx b/studio/frontend/src/features/hub/hub-page.tsx index 7965c6b2da..714a3023ed 100644 --- a/studio/frontend/src/features/hub/hub-page.tsx +++ b/studio/frontend/src/features/hub/hub-page.tsx @@ -69,6 +69,7 @@ import type { import { useHubModelVram } from "./hooks/use-hub-model-vram"; import { useModelsSelection } from "./hooks/use-models-selection"; import { useHubInventory } from "./inventory"; +import { LOCAL_MODEL_SOURCE } from "./inventory/constants"; import { CHANNEL_TO_SECTION, type ChannelId, @@ -1309,6 +1310,9 @@ export function ModelsPage() { displayName: ggufVariant ? `${leaf} ยท ${ggufVariant}` : leaf, ggufVariant, isGguf: row.isGguf, + apiLoadable: + row.isGguf && + (row.kind !== "local" || row.source !== LOCAL_MODEL_SOURCE.OLLAMA), meta: { source: "local", isLora: row.modelFormat === "adapter", diff --git a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts index 3cfa571db3..730963cb34 100644 --- a/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts +++ b/studio/frontend/src/features/model-picker/api/migrate-model-overrides.ts @@ -80,7 +80,8 @@ export async function backfillModelOverrides(): Promise { // that test despite being exactly what auto-switch does resolve; the flag // is then set and its settings stay browser-only for good. (entry) => - (entry.ggufVariant != null || entry.modelId.toLowerCase().endsWith(".gguf")) && + (entry.ggufVariant != null || + entry.modelId.toLowerCase().endsWith(".gguf")) && !isDefaultConfig(entry.config), ); if (local.length === 0) { @@ -110,8 +111,26 @@ export async function backfillModelOverrides(): Promise { if (known.has(key)) { continue; } + // Re-read rather than trusting the snapshot taken before the fetch above. + // A save or a forget during that round trip would otherwise be undone by + // this write, since it is queued behind the interactive one and commits + // last: the browser would show the new settings while an API load applied + // the old ones. + const current = listPerModelConfigs().find( + (candidate) => + normalizedOverrideKey( + modelOverrideKey(candidate.modelId, candidate.ggufVariant), + ) === key, + ); + if (!current || isDefaultConfig(current.config)) { + continue; + } try { - await putModelOverride(entry.modelId, entry.ggufVariant, entry.config); + await putModelOverride( + current.modelId, + current.ggufVariant, + current.config, + ); } catch { failed = true; } diff --git a/studio/frontend/src/features/model-picker/components/model-config-page.tsx b/studio/frontend/src/features/model-picker/components/model-config-page.tsx index ec02532217..adf97553df 100644 --- a/studio/frontend/src/features/model-picker/components/model-config-page.tsx +++ b/studio/frontend/src/features/model-picker/components/model-config-page.tsx @@ -32,6 +32,7 @@ import { useRef, useState, } from "react"; +import { syncModelOverride } from "../api/model-overrides"; import { useDefaultChatTemplate, useModelMaxPositionEmbeddings, @@ -55,7 +56,6 @@ import { resolveInitialConfig, savePerModelConfig, } from "../model-config/per-model-config"; -import { syncModelOverride } from "../api/model-overrides"; import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog"; import type { ModelPickTarget } from "./model-selector/types"; import { @@ -74,14 +74,16 @@ const SELECT_TRIGGER_CLASS = `grid h-8 min-w-0 grid-cols-[minmax(0,1fr)_auto] it const NUMBER_INPUT_CLASS = `h-8 w-[92px] ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-right text-ui-13 font-medium text-nav-fg outline-none focus-visible:ring-0`; const KV_CACHE_DTYPE_DEFAULT = "f16"; -const SPECULATIVE_TYPE_LABELS: Record<(typeof SPECULATIVE_TYPES)[number], string> = - { - auto: "Auto", - mtp: "MTP", - ngram: "Ngram", - "mtp+ngram": "MTP+Ngram", - off: "Off", - }; +const SPECULATIVE_TYPE_LABELS: Record< + (typeof SPECULATIVE_TYPES)[number], + string +> = { + auto: "Auto", + mtp: "MTP", + ngram: "Ngram", + "mtp+ngram": "MTP+Ngram", + off: "Off", +}; function hasNonDefaultAdvanced(config: PerModelConfig): boolean { return ( @@ -352,8 +354,8 @@ function GpuMemorySettings({ info={ <> Layers to keep on the GPU (--gpu-layers); the rest run on CPU. - Auto lets llama.cpp size the split (and the context) to fit VRAM. - At the maximum, the whole model is on the GPU. + Auto lets llama.cpp size the split (and the context) to fit + VRAM. At the maximum, the whole model is on the GPU. } /> @@ -738,8 +740,7 @@ export function ModelConfigPage({ ), maxContext, ); - const setContextLength = (v: number) => - update({ customContextLength: v }); + const setContextLength = (v: number) => update({ customContextLength: v }); const baseline = loadedConfig ?? DEFAULT_PER_MODEL_CONFIG; const atBaseline = perModelConfigsEqual(config, baseline); // An explicit customContextLength equal to the native ceiling is still an @@ -888,10 +889,11 @@ export function ModelConfigPage({ // Skipped when the local write failed (quota, a future-schema entry): the // browser and the server would otherwise permanently disagree about this // model, with no way for the user to tell which one the next load used. - // GGUF only: the API auto-switch resolver indexes GGUFs, so mirroring a - // safetensors config to the server would advertise settings on the monitor's - // "applied on API load" list that no API request can ever apply. - if (!saveFailed && target.isGguf) { + // Auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and + // skips Ollama's scanner, so mirroring either a safetensors config or an + // Ollama one would advertise settings on the monitor's "applied on API load" + // list that no API request can ever apply. + if (!saveFailed && (target.apiLoadable ?? target.isGguf)) { syncModelOverride( target.id, target.ggufVariant, diff --git a/studio/frontend/src/features/model-picker/components/model-selector/types.ts b/studio/frontend/src/features/model-picker/components/model-selector/types.ts index 9adf2d899e..e92a7f3e57 100644 --- a/studio/frontend/src/features/model-picker/components/model-selector/types.ts +++ b/studio/frontend/src/features/model-picker/components/model-selector/types.ts @@ -49,6 +49,15 @@ export interface ModelPickTarget { displayName: string; ggufVariant?: string | null; isGguf: boolean; + /** + * Whether an OpenAI-compatible request can actually load this model. + * + * Not the same as isGguf: local_model_resolver skips Ollama's scanner, so an + * Ollama GGUF is never in the auto-switch index and no API request can resolve + * it. Mirroring its settings would advertise a load that cannot happen. + * Defaults to isGguf where a caller does not know. + */ + apiLoadable?: boolean; meta: ModelSelectorChangeMeta; } diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 8d0406caca..bc70549b1d 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -660,7 +660,7 @@ def test_only_gguf_configs_are_mirrored_to_the_server(): loads safetensors models and must honour their config. """ src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split()) - assert "if (!saveFailed && target.isGguf) { syncModelOverride(" in src + assert "if (!saveFailed && (target.apiLoadable ?? target.isGguf)) { syncModelOverride(" in src # The local save is not behind the same gate. assert "if (remember) { saveFailed = !savePerModelConfig(" in src @@ -731,7 +731,7 @@ def test_api_reach_copy_is_limited_to_gguf_models(): apply to an API request describes a load that cannot happen. """ src = " ".join(_read("features/hub/catalog/hub-model-settings-view.tsx").split()) - assert "{target.isGguf ?" in src + assert "{(target.apiLoadable ?? target.isGguf)" in src assert "Saved settings apply everywhere Studio loads this model." in src @@ -822,3 +822,23 @@ def test_a_failed_detail_fetch_is_retried(): assert "const DETAIL_FETCH_ATTEMPTS = 3;" in src assert "const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_);" in src assert "if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) { return; }" in src + + +def test_ollama_models_are_not_advertised_as_api_loadable(): + """local_model_resolver skips Ollama's scanner, so an Ollama GGUF is never in + the auto-switch index and no OpenAI request can resolve it. target.isGguf is + still true for one, so gating on that alone mirrored settings the API can + never apply and told the user the opposite.""" + types_src = " ".join( + _read("features/model-picker/components/model-selector/types.ts").split() + ) + assert "apiLoadable?: boolean;" in types_src + hub = " ".join(_read("features/hub/hub-page.tsx").split()) + assert 'row.source !== LOCAL_MODEL_SOURCE.OLLAMA' in hub + assert 'apiLoadable:' in hub + backend = ( + WORKDIR / "studio" / "backend" / "core" / "inference" / "local_model_resolver.py" + ).read_text(encoding = "utf-8") + assert "Ollama's\n scanner is skipped" in backend or "scanner is skipped" in backend, ( + "the rule this mirrors" + )