Re-read local state before backfilling, and stop advertising Ollama as API loadable
The backfill wrote the snapshot it took before fetchModelOverrides resolved, so a save or a forget during that round trip was undone: the write is queued behind the interactive one and commits last, leaving the browser showing the new settings while an API load applied the old ones. Each write now re-reads the model's current local config and skips it if it has gone or gone back to defaults. Verified against the real module under node: the write carries maxSeqLength 9999 where it previously carried the stale 1000. target.isGguf was also standing in for "an API request can load this". It cannot for an Ollama model: local_model_resolver skips Ollama's scanner on purpose, so those models are never in the auto-switch index, yet the mirror ran and the settings page told the user the API would apply them. The target now carries apiLoadable, set from the inventory source the row already has, and both the mirror and that sentence read it.
This commit is contained in:
danielhanchen
parent
2b2705bcdc
commit
748b531528
6 changed files with 80 additions and 26 deletions
|
|
@ -10,9 +10,9 @@
|
|||
|
||||
import { ModelConfigPage, type ModelPickTarget } from "@/features/model-picker";
|
||||
import type { PerModelConfig } from "@/features/model-picker";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { ArrowLeft01Icon, Globe02Icon } from "@hugeicons/core-free-icons";
|
||||
import { HugeiconsIcon } from "@hugeicons/react";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
|
||||
export function HubModelSettingsView({
|
||||
|
|
@ -109,10 +109,10 @@ export function HubModelSettingsView({
|
|||
/>
|
||||
</span>
|
||||
<p className="min-w-0 text-ui-12 leading-[1.5] text-muted-foreground">
|
||||
{/* Only a GGUF is mirrored to the server, because API auto-switch
|
||||
indexes GGUFs only, so promising the API case for anything
|
||||
else describes a load that cannot happen. */}
|
||||
{target.isGguf
|
||||
{/* Only what auto-switch can reach is mirrored to the server: it
|
||||
indexes GGUFs and skips Ollama, so promising the API case for
|
||||
anything else describes a load that cannot happen. */}
|
||||
{(target.apiLoadable ?? target.isGguf)
|
||||
? "Saved settings apply everywhere this model loads, including when an OpenAI-compatible API request asks for it."
|
||||
: "Saved settings apply everywhere Studio loads this model."}{" "}
|
||||
Turn on{" "}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ import type {
|
|||
import { useHubModelVram } from "./hooks/use-hub-model-vram";
|
||||
import { useModelsSelection } from "./hooks/use-models-selection";
|
||||
import { useHubInventory } from "./inventory";
|
||||
import { LOCAL_MODEL_SOURCE } from "./inventory/constants";
|
||||
import {
|
||||
CHANNEL_TO_SECTION,
|
||||
type ChannelId,
|
||||
|
|
@ -1309,6 +1310,9 @@ export function ModelsPage() {
|
|||
displayName: ggufVariant ? `${leaf} · ${ggufVariant}` : leaf,
|
||||
ggufVariant,
|
||||
isGguf: row.isGguf,
|
||||
apiLoadable:
|
||||
row.isGguf &&
|
||||
(row.kind !== "local" || row.source !== LOCAL_MODEL_SOURCE.OLLAMA),
|
||||
meta: {
|
||||
source: "local",
|
||||
isLora: row.modelFormat === "adapter",
|
||||
|
|
|
|||
|
|
@ -80,7 +80,8 @@ export async function backfillModelOverrides(): Promise<void> {
|
|||
// that test despite being exactly what auto-switch does resolve; the flag
|
||||
// is then set and its settings stay browser-only for good.
|
||||
(entry) =>
|
||||
(entry.ggufVariant != null || entry.modelId.toLowerCase().endsWith(".gguf")) &&
|
||||
(entry.ggufVariant != null ||
|
||||
entry.modelId.toLowerCase().endsWith(".gguf")) &&
|
||||
!isDefaultConfig(entry.config),
|
||||
);
|
||||
if (local.length === 0) {
|
||||
|
|
@ -110,8 +111,26 @@ export async function backfillModelOverrides(): Promise<void> {
|
|||
if (known.has(key)) {
|
||||
continue;
|
||||
}
|
||||
// Re-read rather than trusting the snapshot taken before the fetch above.
|
||||
// A save or a forget during that round trip would otherwise be undone by
|
||||
// this write, since it is queued behind the interactive one and commits
|
||||
// last: the browser would show the new settings while an API load applied
|
||||
// the old ones.
|
||||
const current = listPerModelConfigs().find(
|
||||
(candidate) =>
|
||||
normalizedOverrideKey(
|
||||
modelOverrideKey(candidate.modelId, candidate.ggufVariant),
|
||||
) === key,
|
||||
);
|
||||
if (!current || isDefaultConfig(current.config)) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await putModelOverride(entry.modelId, entry.ggufVariant, entry.config);
|
||||
await putModelOverride(
|
||||
current.modelId,
|
||||
current.ggufVariant,
|
||||
current.config,
|
||||
);
|
||||
} catch {
|
||||
failed = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import {
|
|||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { syncModelOverride } from "../api/model-overrides";
|
||||
import {
|
||||
useDefaultChatTemplate,
|
||||
useModelMaxPositionEmbeddings,
|
||||
|
|
@ -55,7 +56,6 @@ import {
|
|||
resolveInitialConfig,
|
||||
savePerModelConfig,
|
||||
} from "../model-config/per-model-config";
|
||||
import { syncModelOverride } from "../api/model-overrides";
|
||||
import { ChatTemplateEditorDialog } from "./chat-template-editor-dialog";
|
||||
import type { ModelPickTarget } from "./model-selector/types";
|
||||
import {
|
||||
|
|
@ -74,14 +74,16 @@ const SELECT_TRIGGER_CLASS = `grid h-8 min-w-0 grid-cols-[minmax(0,1fr)_auto] it
|
|||
const NUMBER_INPUT_CLASS = `h-8 w-[92px] ${CONTROL_SURFACE} pl-3 pr-2 py-0 text-right text-ui-13 font-medium text-nav-fg outline-none focus-visible:ring-0`;
|
||||
|
||||
const KV_CACHE_DTYPE_DEFAULT = "f16";
|
||||
const SPECULATIVE_TYPE_LABELS: Record<(typeof SPECULATIVE_TYPES)[number], string> =
|
||||
{
|
||||
auto: "Auto",
|
||||
mtp: "MTP",
|
||||
ngram: "Ngram",
|
||||
"mtp+ngram": "MTP+Ngram",
|
||||
off: "Off",
|
||||
};
|
||||
const SPECULATIVE_TYPE_LABELS: Record<
|
||||
(typeof SPECULATIVE_TYPES)[number],
|
||||
string
|
||||
> = {
|
||||
auto: "Auto",
|
||||
mtp: "MTP",
|
||||
ngram: "Ngram",
|
||||
"mtp+ngram": "MTP+Ngram",
|
||||
off: "Off",
|
||||
};
|
||||
|
||||
function hasNonDefaultAdvanced(config: PerModelConfig): boolean {
|
||||
return (
|
||||
|
|
@ -352,8 +354,8 @@ function GpuMemorySettings({
|
|||
info={
|
||||
<>
|
||||
Layers to keep on the GPU (--gpu-layers); the rest run on CPU.
|
||||
Auto lets llama.cpp size the split (and the context) to fit VRAM.
|
||||
At the maximum, the whole model is on the GPU.
|
||||
Auto lets llama.cpp size the split (and the context) to fit
|
||||
VRAM. At the maximum, the whole model is on the GPU.
|
||||
</>
|
||||
}
|
||||
/>
|
||||
|
|
@ -738,8 +740,7 @@ export function ModelConfigPage({
|
|||
),
|
||||
maxContext,
|
||||
);
|
||||
const setContextLength = (v: number) =>
|
||||
update({ customContextLength: v });
|
||||
const setContextLength = (v: number) => update({ customContextLength: v });
|
||||
const baseline = loadedConfig ?? DEFAULT_PER_MODEL_CONFIG;
|
||||
const atBaseline = perModelConfigsEqual(config, baseline);
|
||||
// An explicit customContextLength equal to the native ceiling is still an
|
||||
|
|
@ -888,10 +889,11 @@ export function ModelConfigPage({
|
|||
// Skipped when the local write failed (quota, a future-schema entry): the
|
||||
// browser and the server would otherwise permanently disagree about this
|
||||
// model, with no way for the user to tell which one the next load used.
|
||||
// GGUF only: the API auto-switch resolver indexes GGUFs, so mirroring a
|
||||
// safetensors config to the server would advertise settings on the monitor's
|
||||
// "applied on API load" list that no API request can ever apply.
|
||||
if (!saveFailed && target.isGguf) {
|
||||
// Auto-switch reach, not just GGUF-ness: the resolver indexes GGUFs and
|
||||
// skips Ollama's scanner, so mirroring either a safetensors config or an
|
||||
// Ollama one would advertise settings on the monitor's "applied on API load"
|
||||
// list that no API request can ever apply.
|
||||
if (!saveFailed && (target.apiLoadable ?? target.isGguf)) {
|
||||
syncModelOverride(
|
||||
target.id,
|
||||
target.ggufVariant,
|
||||
|
|
|
|||
|
|
@ -49,6 +49,15 @@ export interface ModelPickTarget {
|
|||
displayName: string;
|
||||
ggufVariant?: string | null;
|
||||
isGguf: boolean;
|
||||
/**
|
||||
* Whether an OpenAI-compatible request can actually load this model.
|
||||
*
|
||||
* Not the same as isGguf: local_model_resolver skips Ollama's scanner, so an
|
||||
* Ollama GGUF is never in the auto-switch index and no API request can resolve
|
||||
* it. Mirroring its settings would advertise a load that cannot happen.
|
||||
* Defaults to isGguf where a caller does not know.
|
||||
*/
|
||||
apiLoadable?: boolean;
|
||||
meta: ModelSelectorChangeMeta;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -660,7 +660,7 @@ def test_only_gguf_configs_are_mirrored_to_the_server():
|
|||
loads safetensors models and must honour their config.
|
||||
"""
|
||||
src = " ".join(_read("features/model-picker/components/model-config-page.tsx").split())
|
||||
assert "if (!saveFailed && target.isGguf) { syncModelOverride(" in src
|
||||
assert "if (!saveFailed && (target.apiLoadable ?? target.isGguf)) { syncModelOverride(" in src
|
||||
# The local save is not behind the same gate.
|
||||
assert "if (remember) { saveFailed = !savePerModelConfig(" in src
|
||||
|
||||
|
|
@ -731,7 +731,7 @@ def test_api_reach_copy_is_limited_to_gguf_models():
|
|||
apply to an API request describes a load that cannot happen.
|
||||
"""
|
||||
src = " ".join(_read("features/hub/catalog/hub-model-settings-view.tsx").split())
|
||||
assert "{target.isGguf ?" in src
|
||||
assert "{(target.apiLoadable ?? target.isGguf)" in src
|
||||
assert "Saved settings apply everywhere Studio loads this model." in src
|
||||
|
||||
|
||||
|
|
@ -822,3 +822,23 @@ def test_a_failed_detail_fetch_is_retried():
|
|||
assert "const DETAIL_FETCH_ATTEMPTS = 3;" in src
|
||||
assert "const detailInFlight = selectedId_ != null && loadingDetails.has(selectedId_);" in src
|
||||
assert "if (attemptsRef.current.count >= DETAIL_FETCH_ATTEMPTS) { return; }" in src
|
||||
|
||||
|
||||
def test_ollama_models_are_not_advertised_as_api_loadable():
|
||||
"""local_model_resolver skips Ollama's scanner, so an Ollama GGUF is never in
|
||||
the auto-switch index and no OpenAI request can resolve it. target.isGguf is
|
||||
still true for one, so gating on that alone mirrored settings the API can
|
||||
never apply and told the user the opposite."""
|
||||
types_src = " ".join(
|
||||
_read("features/model-picker/components/model-selector/types.ts").split()
|
||||
)
|
||||
assert "apiLoadable?: boolean;" in types_src
|
||||
hub = " ".join(_read("features/hub/hub-page.tsx").split())
|
||||
assert 'row.source !== LOCAL_MODEL_SOURCE.OLLAMA' in hub
|
||||
assert 'apiLoadable:' in hub
|
||||
backend = (
|
||||
WORKDIR / "studio" / "backend" / "core" / "inference" / "local_model_resolver.py"
|
||||
).read_text(encoding = "utf-8")
|
||||
assert "Ollama's\n scanner is skipped" in backend or "scanner is skipped" in backend, (
|
||||
"the rule this mirrors"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue