From cc3f0430e6e62a1cb0b720de11d66db538046bfb Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:01:06 -0700 Subject: [PATCH 01/14] Studio: detect local MTP subdirectory drafters --- .../tests/test_mtp_drafter_companion.py | 47 +++++++++++++++ studio/backend/utils/models/model_config.py | 58 ++++++++++++++++++- .../src/features/chat/chat-settings-sheet.tsx | 6 +- 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 02230632b6..999e1bd0ca 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -211,6 +211,53 @@ def test_detect_mtp_file_search_root(tmp_path): assert found is not None and found.endswith("mtp-gemma-4-12b-it.gguf") +def test_detect_mtp_file_falls_back_to_new_scheme_subdir(tmp_path): + weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + (sub / "mtp-gemma-4-E4B-it-BF16.gguf").write_bytes(b"x") + q4 = sub / "mtp-gemma-4-E4B-it-Q4_0.gguf" + q4.write_bytes(b"x") + + found = detect_mtp_file(str(weight)) + assert found == str(q4.resolve()) + + +def test_detect_mtp_file_falls_back_to_old_scheme_subdir(tmp_path): + weight = tmp_path / "gemma-4-12b-it-Q4_K_M.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + drafter = sub / "gemma-4-12b-it-Q8_0-MTP.gguf" + drafter.write_bytes(b"x") + + found = detect_mtp_file(str(weight)) + assert found == str(drafter.resolve()) + + +def test_detect_mtp_file_root_still_wins_over_subdir(tmp_path): + weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + root = tmp_path / "mtp-gemma-4-E4B-it.gguf" + root.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + (sub / "mtp-gemma-4-E4B-it-Q4_0.gguf").write_bytes(b"x") + + assert detect_mtp_file(str(weight)) == str(root.resolve()) + + +def test_detect_mtp_file_subdir_skips_foreign_drafter(tmp_path): + weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + (sub / "mtp-gemma-4-12b-it-Q4_0.gguf").write_bytes(b"x") + + assert detect_mtp_file(str(weight)) is None + + # ── Reload dedup includes the drafter ──────────────────────────────── diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 4897f05ce4..6fa1b85c08 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1455,8 +1455,36 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st unsloth names the drafter ``mtp-.gguf`` where ```` prefixes the weight filename across all Gemma 4 repos (e.g. ``mtp-gemma-4-12B-it.gguf`` next to ``gemma-4-12B-it-qat-Q4_0.gguf``). - An unmatched drafter is skipped (fail-safe: no MTP). + If the root drafter is absent, also accept its precision copy under the + repository's ``MTP/`` directory. An unmatched drafter is skipped. """ + + def _pairing_stem(name: str) -> str: + stem = Path(name).stem.lower() + if stem.startswith("mtp-"): + stem = stem[len("mtp-") :] + if stem.endswith("-mtp"): + stem = stem[: -len("-mtp")] + return re.sub(r"-(?:q[0-9]+_[0-9]+|bf16|f16)$", "", stem) + + def _matches_weight(candidate: Path) -> bool: + if weight_name is None: + return True + stem = _pairing_stem(candidate.name) + return bool(stem) and weight_name.startswith(stem) + + def _precision_rank(candidate: Path) -> tuple[int, str]: + name = candidate.name.lower() + if "-q4_0" in name: + rank = 0 + elif "-q8_0" in name: + rank = 1 + elif "-bf16" in name or "-f16" in name: + rank = 2 + else: + rank = 3 + return rank, name + p = Path(path) weight_name = p.name.lower() if p.suffix.lower() == ".gguf" else None start_dir = p.parent if p.is_file() else p @@ -1472,14 +1500,38 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st name = f.name.lower() if not (name.startswith("mtp-") and name.endswith(".gguf")): continue - stem = name[len("mtp-") : -len(".gguf")] - if not stem or (weight_name is not None and not weight_name.startswith(stem)): + if not _matches_weight(f): continue try: if f.is_file(): return str(f.resolve()) except OSError: continue + + subdir_candidates: list[Path] = [] + for d in dirs: + mtp_dir = d / "MTP" + try: + entries = sorted(mtp_dir.iterdir()) + except OSError: + continue + for f in entries: + rel = f"MTP/{f.name}" + if not _is_mtp_drafter(rel) or not _matches_weight(f): + continue + try: + if f.is_file(): + subdir_candidates.append(f) + except OSError: + continue + + for candidate in sorted(subdir_candidates, key = _precision_rank): + try: + resolved = candidate.resolve() + except OSError: + continue + logger.info(f"Detected MTP subdirectory drafter: {resolved}") + return str(resolved) return None diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index c3a59e9860..e52a6e15cc 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -378,6 +378,8 @@ export function ChatSettingsPanel({ const isMobile = useIsMobile(); const isLoadedGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; const currentCheckpoint = params.checkpoint; + const isDirectLocalGguf = + currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false; const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); // Direct-file / custom-folder GGUFs load without a variant label but still // report a GGUF context, so detect them via the context and the checkpoint @@ -812,7 +814,9 @@ export function ChatSettingsPanel({ : specFallbackReason === "runtime_error" ? "MTP could not start for this model on the installed llama.cpp build, so it is running without speculative decoding." : specFallbackReason === "drafter_not_found" - ? "This model supports MTP, but its drafter file could not be downloaded, so MTP is off and it falls back to n-gram speculative decoding where the llama.cpp build supports it. Check your network connection or Hugging Face access, then reload the model to retry the drafter." + ? isDirectLocalGguf + ? "This local model supports MTP, but no matching drafter file was found. Place its mtp-*.gguf beside the model or in its MTP folder, then reload the model." + : "This model supports MTP, but its drafter file could not be downloaded, so MTP is off and it falls back to n-gram speculative decoding where the llama.cpp build supports it. Check your network connection or Hugging Face access, then reload the model to retry the drafter." : `MTP is not available in the installed llama.cpp build, so this model is running without it.${ llamaUpdateStatus?.update_available ? " Update llama.cpp to enable it." From 8ef7c9c345ed194d7f60d2c418be7b5c4eec6bc7 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Fri, 24 Jul 2026 04:37:56 -0700 Subject: [PATCH 02/14] Studio: allow native MTP subdirectory companions --- studio/backend/routes/inference.py | 26 +++- .../tests/test_mtp_drafter_companion.py | 56 +++++++++ .../tests/test_native_gguf_companion.py | 119 ++++++++++++++++++ studio/backend/utils/models/model_config.py | 27 ++-- studio/backend/utils/native_path_leases.py | 16 +++ 5 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 studio/backend/tests/test_native_gguf_companion.py diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 445a26f04d..bc48f0b0bf 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1027,6 +1027,7 @@ try: NativePathLeaseError, display_label_for_native_path, is_registered_native_path_label, + native_gguf_companion_parent_allowed, redact_native_paths, verify_native_path_lease, ) @@ -1065,6 +1066,7 @@ except ImportError: NativePathLeaseError, display_label_for_native_path, is_registered_native_path_label, + native_gguf_companion_parent_allowed, redact_native_paths, verify_native_path_lease, ) @@ -3070,11 +3072,15 @@ def _monitor_active_model() -> Optional[str]: def _validate_native_gguf_companion( - companion_path: str | None, gguf_path: str | None, label: str + companion_path: str | None, + gguf_path: str | None, + label: str, + *, + allow_mtp_subdir: bool = False, ) -> None: """Reject a companion GGUF (mmproj / MTP drafter) that a native-lease load would otherwise hand to llama-server: must be a regular file (no symlink - escaping the leased directory) living next to the selected GGUF.""" + escaping the leased directory) in a permitted location.""" if not companion_path or not gguf_path: return import stat as _stat_module @@ -3096,10 +3102,17 @@ def _validate_native_gguf_companion( detail = f"Native {label} must be a regular file.", ) try: - if companion.resolve(strict = True).parent != gguf.resolve(strict = True).parent: + if not native_gguf_companion_parent_allowed( + companion, gguf, allow_mtp_subdir = allow_mtp_subdir + ): + location = ( + "beside the selected GGUF or in its MTP directory" + if allow_mtp_subdir + else "next to the selected GGUF" + ) raise HTTPException( status_code = 400, - detail = f"Native {label} must live next to the selected GGUF.", + detail = f"Native {label} must live {location}.", ) except OSError as exc: raise HTTPException( @@ -4644,7 +4657,10 @@ async def _load_model_impl( # model): drop it rather than fail the load. try: _validate_native_gguf_companion( - config.gguf_mtp_file, config.gguf_file, "MTP drafter" + config.gguf_mtp_file, + config.gguf_file, + "MTP drafter", + allow_mtp_subdir = True, ) except HTTPException as exc: logger.warning("Dropping MTP drafter for native load: %s", exc.detail) diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 999e1bd0ca..ed8a6a5d0e 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -34,6 +34,7 @@ from utils.models.model_config import ( detect_mtp_file, extract_model_size_b, ) +from utils.native_path_leases import native_gguf_companion_parent_allowed # ── Predicate + layering mirrors ───────────────────────────────────── @@ -258,6 +259,61 @@ def test_detect_mtp_file_subdir_skips_foreign_drafter(tmp_path): assert detect_mtp_file(str(weight)) is None +def test_detect_mtp_file_accepts_case_variant_subdir(tmp_path): + weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "mtp" + sub.mkdir() + drafter = sub / "mtp-gemma-4-E4B-it-Q4_0.gguf" + drafter.write_bytes(b"x") + + assert detect_mtp_file(str(weight)) == str(drafter.resolve()) + + +def test_native_companion_parent_accepts_root_and_mtp_subdir(tmp_path): + weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + root_drafter = tmp_path / "mtp-gemma-4-E4B-it.gguf" + root_drafter.write_bytes(b"x") + sub = tmp_path / "MtP" + sub.mkdir() + nested_drafter = sub / "mtp-gemma-4-E4B-it-Q4_0.gguf" + nested_drafter.write_bytes(b"x") + + assert native_gguf_companion_parent_allowed(root_drafter, weight) + assert native_gguf_companion_parent_allowed(nested_drafter, weight, allow_mtp_subdir = True) + + +def test_native_companion_parent_rejects_other_nested_directory(tmp_path): + weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "other" + sub.mkdir() + drafter = sub / "mtp-gemma-4-E4B-it-Q4_0.gguf" + drafter.write_bytes(b"x") + + assert not native_gguf_companion_parent_allowed(drafter, weight) + + +def test_native_companion_parent_rejects_mtp_symlink_escape(tmp_path): + model_dir = tmp_path / "model" + model_dir.mkdir() + weight = model_dir / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + outside = tmp_path / "outside" + outside.mkdir() + drafter = outside / "mtp-gemma-4-E4B-it-Q4_0.gguf" + drafter.write_bytes(b"x") + try: + (model_dir / "MTP").symlink_to(outside, target_is_directory = True) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + assert not native_gguf_companion_parent_allowed( + model_dir / "MTP" / drafter.name, weight, allow_mtp_subdir = True + ) + + # ── Reload dedup includes the drafter ──────────────────────────────── diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py new file mode 100644 index 0000000000..d4fe8ba831 --- /dev/null +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Native GGUF companion path validation.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +from fastapi import HTTPException + +_BACKEND_DIR = str(Path(__file__).resolve().parent.parent) +if _BACKEND_DIR not in sys.path: + sys.path.insert(0, _BACKEND_DIR) + +from routes.inference import _validate_native_gguf_companion + + +def _write_pair(tmp_path: Path, folder: str | None = None) -> tuple[Path, Path]: + weight = tmp_path / "model.gguf" + weight.write_bytes(b"model") + parent = tmp_path if folder is None else tmp_path / folder + parent.mkdir(parents = True, exist_ok = True) + companion = parent / "mtp-model.gguf" + companion.write_bytes(b"draft") + return weight, companion + + +def test_native_companion_allows_model_directory(tmp_path): + weight, companion = _write_pair(tmp_path) + _validate_native_gguf_companion(str(companion), str(weight), "vision companion") + + +@pytest.mark.parametrize("folder", ["MTP", "mtp", "MtP"]) +def test_native_mtp_companion_allows_mtp_directory(tmp_path, folder): + weight, companion = _write_pair(tmp_path, folder) + _validate_native_gguf_companion( + str(companion), str(weight), "MTP drafter", allow_mtp_subdir = True + ) + + +def test_native_vision_companion_rejects_mtp_directory(tmp_path): + weight, companion = _write_pair(tmp_path, "MTP") + with pytest.raises(HTTPException, match = "must live next to"): + _validate_native_gguf_companion(str(companion), str(weight), "vision companion") + + +@pytest.mark.parametrize("folder", ["other", "MTP/deeper", "mtp/deeper"]) +def test_native_companion_rejects_arbitrary_nesting(tmp_path, folder): + weight, companion = _write_pair(tmp_path, folder) + with pytest.raises(HTTPException, match = "must live beside") as error: + _validate_native_gguf_companion( + str(companion), str(weight), "MTP drafter", allow_mtp_subdir = True + ) + assert error.value.status_code == 400 + + +def test_native_companion_rejects_file_symlink(tmp_path): + weight, companion = _write_pair(tmp_path) + link = tmp_path / "mtp-link.gguf" + try: + link.symlink_to(companion) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + with pytest.raises(HTTPException, match = "regular file"): + _validate_native_gguf_companion(str(link), str(weight), "MTP drafter") + + +def test_native_companion_rejects_directory_symlink_escape(tmp_path): + model_dir = tmp_path / "model" + outside = tmp_path / "outside" + model_dir.mkdir() + outside.mkdir() + weight = model_dir / "model.gguf" + weight.write_bytes(b"model") + companion = outside / "mtp-model.gguf" + companion.write_bytes(b"draft") + try: + (model_dir / "MTP").symlink_to(outside, target_is_directory = True) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + with pytest.raises(HTTPException, match = "must live beside"): + _validate_native_gguf_companion( + str(model_dir / "MTP" / companion.name), + str(weight), + "MTP drafter", + allow_mtp_subdir = True, + ) + + +def test_native_companion_rejects_missing_file(tmp_path): + weight = tmp_path / "model.gguf" + weight.write_bytes(b"model") + with pytest.raises(HTTPException, match = "no longer accessible"): + _validate_native_gguf_companion(str(tmp_path / "missing.gguf"), str(weight), "MTP drafter") + + +def test_native_companion_rejects_directory(tmp_path): + weight = tmp_path / "model.gguf" + weight.write_bytes(b"model") + companion = tmp_path / "mtp-model.gguf" + companion.mkdir() + with pytest.raises(HTTPException, match = "regular file"): + _validate_native_gguf_companion(str(companion), str(weight), "MTP drafter") + + +def test_native_companion_rejects_missing_weight(tmp_path): + companion = tmp_path / "mtp-model.gguf" + companion.write_bytes(b"draft") + with pytest.raises(HTTPException, match = "no longer accessible"): + _validate_native_gguf_companion( + str(companion), str(tmp_path / "missing.gguf"), "MTP drafter" + ) + + +def test_native_companion_none_is_noop(): + _validate_native_gguf_companion(None, None, "MTP drafter") diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 6fa1b85c08..4c71414559 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1510,20 +1510,33 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st subdir_candidates: list[Path] = [] for d in dirs: - mtp_dir = d / "MTP" try: - entries = sorted(mtp_dir.iterdir()) + parent_entries = sorted(d.iterdir()) except OSError: continue - for f in entries: - rel = f"MTP/{f.name}" - if not _is_mtp_drafter(rel) or not _matches_weight(f): + mtp_dirs: list[Path] = [] + for entry in parent_entries: + if entry.name.casefold() != "mtp": continue try: - if f.is_file(): - subdir_candidates.append(f) + if entry.is_dir(): + mtp_dirs.append(entry) except OSError: continue + for mtp_dir in mtp_dirs: + try: + entries = sorted(mtp_dir.iterdir()) + except OSError: + continue + for f in entries: + rel = f"MTP/{f.name}" + if not _is_mtp_drafter(rel) or not _matches_weight(f): + continue + try: + if f.is_file(): + subdir_candidates.append(f) + except OSError: + continue for candidate in sorted(subdir_candidates, key = _precision_rank): try: diff --git a/studio/backend/utils/native_path_leases.py b/studio/backend/utils/native_path_leases.py index 3ed7faa7c2..090cedfeb6 100644 --- a/studio/backend/utils/native_path_leases.py +++ b/studio/backend/utils/native_path_leases.py @@ -47,6 +47,22 @@ class NativePathLeaseError(ValueError): """Raised when a native path grant is missing, invalid, or unsafe.""" +def native_gguf_companion_parent_allowed( + companion_path: str | Path, + gguf_path: str | Path, + *, + allow_mtp_subdir: bool = False, +) -> bool: + """Check whether a GGUF companion is in an allowed directory.""" + companion_parent = Path(companion_path).resolve(strict = True).parent + gguf_parent = Path(gguf_path).resolve(strict = True).parent + return companion_parent == gguf_parent or bool( + allow_mtp_subdir + and companion_parent.parent == gguf_parent + and companion_parent.name.casefold() == "mtp" + ) + + @dataclass(frozen = True) class NativePathGrant: operation: str From e420e8411555aaafeb1a0ed915a98509c77d85d4 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Fri, 24 Jul 2026 05:07:15 -0700 Subject: [PATCH 03/14] Studio: classify local GGUF fallback guidance --- .../src/features/chat/chat-settings-sheet.tsx | 17 +++++++++++++---- tests/studio/test_model_picker_contracts.py | 12 ++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index e52a6e15cc..e2303d927e 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -88,7 +88,10 @@ import { providerSupportsBuiltinCodeExecution, providerSupportsFastMode, } from "./provider-capabilities"; -import { useChatRuntimeStore } from "./stores/chat-runtime-store"; +import { + isLocalModelPath, + useChatRuntimeStore, +} from "./stores/chat-runtime-store"; import type { InferenceParams } from "./types/runtime"; export { defaultInferenceParams, type Preset } from "./presets/preset-policy"; @@ -378,8 +381,9 @@ export function ChatSettingsPanel({ const isMobile = useIsMobile(); const isLoadedGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; const currentCheckpoint = params.checkpoint; - const isDirectLocalGguf = - currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false; + const activeNativePathToken = useChatRuntimeStore( + (s) => s.activeNativePathToken, + ); const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); // Direct-file / custom-folder GGUFs load without a variant label but still // report a GGUF context, so detect them via the context and the checkpoint @@ -389,6 +393,11 @@ export function ChatSettingsPanel({ isLoadedGguf || ggufContextLength != null || (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false); + const isLocalGguf = + isGguf && + (activeNativePathToken != null || + isLocalModelPath(currentCheckpoint ?? "") || + (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false)); const ggufMaxContextLength = useChatRuntimeStore( (s) => s.ggufMaxContextLength, ); @@ -814,7 +823,7 @@ export function ChatSettingsPanel({ : specFallbackReason === "runtime_error" ? "MTP could not start for this model on the installed llama.cpp build, so it is running without speculative decoding." : specFallbackReason === "drafter_not_found" - ? isDirectLocalGguf + ? isLocalGguf ? "This local model supports MTP, but no matching drafter file was found. Place its mtp-*.gguf beside the model or in its MTP folder, then reload the model." : "This model supports MTP, but its drafter file could not be downloaded, so MTP is off and it falls back to n-gram speculative decoding where the llama.cpp build supports it. Check your network connection or Hugging Face access, then reload the model to retry the drafter." : `MTP is not available in the installed llama.cpp build, so this model is running without it.${ diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 62b1a3cf76..1dc9761c12 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -318,6 +318,18 @@ def test_local_gguf_diagnostics_gate_on_broad_is_gguf(): assert vram and "isGguf &&" in vram.group(0) and "isLoadedGguf" not in vram.group(0) +def test_local_mtp_warning_covers_path_and_native_gguf_sources(): + """The local MTP recovery text must cover direct files, custom folders, + and native-picker labels instead of classifying only .gguf suffixes.""" + src = _read("features/chat/chat-settings-sheet.tsx") + local = re.search(r"const isLocalGguf =.*?;", src, re.S) + assert local + assert "isGguf &&" in local.group(0) + assert "activeNativePathToken" in local.group(0) + assert "isLocalModelPath" in local.group(0) + assert "isLocalGguf" in src.split('specFallbackReason === "drafter_not_found"', 1)[1] + + def test_fixed_layer_gguf_pins_displayed_context(): """An already-loaded auto-fit GGUF saved with Manual fixed GPU layers must pin the shown context, so a later fresh load keeps the fitted placement From af4464a0e0492e6bb3a37bb255b2b14bf1c165dc Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:41:34 -0700 Subject: [PATCH 04/14] Studio: harden MTP companion pairing --- studio/backend/routes/inference.py | 17 +++++- .../tests/test_mtp_drafter_companion.py | 14 +++++ .../tests/test_native_gguf_companion.py | 58 +++++++++++++++++++ studio/backend/utils/models/model_config.py | 6 +- studio/backend/utils/native_path_leases.py | 16 +++-- 5 files changed, 103 insertions(+), 8 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index bc48f0b0bf..f4b8a94af5 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1020,6 +1020,7 @@ try: from utils.models import ModelConfig from utils.inference import load_inference_config from utils.models.model_config import ( + _local_gguf_companion_search_root, detect_mtp_file, load_model_defaults, ) @@ -1059,6 +1060,7 @@ except ImportError: from utils.models import ModelConfig from utils.inference import load_inference_config from utils.models.model_config import ( + _local_gguf_companion_search_root, detect_mtp_file, load_model_defaults, ) @@ -3077,6 +3079,7 @@ def _validate_native_gguf_companion( label: str, *, allow_mtp_subdir: bool = False, + mtp_search_root: str | Path | None = None, ) -> None: """Reject a companion GGUF (mmproj / MTP drafter) that a native-lease load would otherwise hand to llama-server: must be a regular file (no symlink @@ -3103,7 +3106,10 @@ def _validate_native_gguf_companion( ) try: if not native_gguf_companion_parent_allowed( - companion, gguf, allow_mtp_subdir = allow_mtp_subdir + companion, + gguf, + allow_mtp_subdir = allow_mtp_subdir, + mtp_search_root = mtp_search_root, ): location = ( "beside the selected GGUF or in its MTP directory" @@ -3337,7 +3343,10 @@ def _request_matches_loaded_settings( else llama_backend.extra_args ) if not _extra_args_set_spec_type(effective_extras): - detected = detect_mtp_file(llama_backend.gguf_path) + companion_root = _local_gguf_companion_search_root( + llama_backend.gguf_path, llama_backend.gguf_path + ) + detected = detect_mtp_file(llama_backend.gguf_path, search_root = companion_root) stored = llama_backend.mtp_draft_path try: detected_resolved = Path(detected).resolve() if detected else None @@ -4656,11 +4665,15 @@ async def _load_model_impl( # The drafter is optional (unlike mmproj for a vision # model): drop it rather than fail the load. try: + mtp_search_root = _local_gguf_companion_search_root( + config.gguf_file, config.gguf_file + ) _validate_native_gguf_companion( config.gguf_mtp_file, config.gguf_file, "MTP drafter", allow_mtp_subdir = True, + mtp_search_root = mtp_search_root, ) except HTTPException as exc: logger.warning("Dropping MTP drafter for native load: %s", exc.detail) diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index ed8a6a5d0e..5c1e7fb035 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -259,6 +259,20 @@ def test_detect_mtp_file_subdir_skips_foreign_drafter(tmp_path): assert detect_mtp_file(str(weight)) is None +@pytest.mark.parametrize( + "companion_path", + ["mtp-gemma-4-E4B-it-Q4_0.gguf", "MTP/mtp-gemma-4-E4B-it-Q4_0.gguf"], +) +def test_detect_mtp_file_requires_model_name_boundary(tmp_path, companion_path): + weight = tmp_path / "gemma-4-E4B-item-qat-Q4_0.gguf" + weight.write_bytes(b"x") + companion = tmp_path / companion_path + companion.parent.mkdir(parents = True, exist_ok = True) + companion.write_bytes(b"x") + + assert detect_mtp_file(str(weight)) is None + + def test_detect_mtp_file_accepts_case_variant_subdir(tmp_path): weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" weight.write_bytes(b"x") diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index d4fe8ba831..9ed260aa90 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -16,9 +16,13 @@ if _BACKEND_DIR not in sys.path: sys.path.insert(0, _BACKEND_DIR) from routes.inference import _validate_native_gguf_companion +from routes.inference import _request_matches_loaded_settings +from core.inference.llama_cpp import LlamaCppBackend +from models.inference import LoadRequest def _write_pair(tmp_path: Path, folder: str | None = None) -> tuple[Path, Path]: + tmp_path.mkdir(parents = True, exist_ok = True) weight = tmp_path / "model.gguf" weight.write_bytes(b"model") parent = tmp_path if folder is None else tmp_path / folder @@ -41,6 +45,60 @@ def test_native_mtp_companion_allows_mtp_directory(tmp_path, folder): ) +def test_native_mtp_companion_allows_repo_root_mtp_directory(tmp_path): + quant_dir = tmp_path / "Q4_0" + weight, _ = _write_pair(quant_dir) + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model.gguf" + companion.write_bytes(b"draft") + + _validate_native_gguf_companion( + str(companion), + str(weight), + "MTP drafter", + allow_mtp_subdir = True, + mtp_search_root = str(tmp_path), + ) + + +def test_native_mtp_companion_rejects_unrelated_search_root(tmp_path): + quant_dir = tmp_path / "repo" / "Q4_0" + weight, _ = _write_pair(quant_dir) + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model.gguf" + companion.write_bytes(b"draft") + + with pytest.raises(HTTPException, match = "must live beside"): + _validate_native_gguf_companion( + str(companion), + str(weight), + "MTP drafter", + allow_mtp_subdir = True, + mtp_search_root = str(tmp_path), + ) + + +def test_reload_dedup_finds_repo_root_mtp_companion(tmp_path, monkeypatch): + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model.gguf" + companion.write_bytes(b"draft") + + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._gguf_path = str(weight) + backend._mtp_draft_path = str(companion) + + request = LoadRequest(model_path = str(weight)) + assert _request_matches_loaded_settings(request, backend) + + def test_native_vision_companion_rejects_mtp_directory(tmp_path): weight, companion = _write_pair(tmp_path, "MTP") with pytest.raises(HTTPException, match = "must live next to"): diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 4c71414559..0d66515182 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1471,7 +1471,11 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st if weight_name is None: return True stem = _pairing_stem(candidate.name) - return bool(stem) and weight_name.startswith(stem) + return ( + bool(stem) + and weight_name.startswith(stem) + and (len(weight_name) == len(stem) or not weight_name[len(stem)].isalnum()) + ) def _precision_rank(candidate: Path) -> tuple[int, str]: name = candidate.name.lower() diff --git a/studio/backend/utils/native_path_leases.py b/studio/backend/utils/native_path_leases.py index 090cedfeb6..d17a96eb00 100644 --- a/studio/backend/utils/native_path_leases.py +++ b/studio/backend/utils/native_path_leases.py @@ -52,15 +52,21 @@ def native_gguf_companion_parent_allowed( gguf_path: str | Path, *, allow_mtp_subdir: bool = False, + mtp_search_root: str | Path | None = None, ) -> bool: """Check whether a GGUF companion is in an allowed directory.""" companion_parent = Path(companion_path).resolve(strict = True).parent gguf_parent = Path(gguf_path).resolve(strict = True).parent - return companion_parent == gguf_parent or bool( - allow_mtp_subdir - and companion_parent.parent == gguf_parent - and companion_parent.name.casefold() == "mtp" - ) + if companion_parent == gguf_parent: + return True + if not allow_mtp_subdir or companion_parent.name.casefold() != "mtp": + return False + allowed_roots = {gguf_parent} + if mtp_search_root is not None: + search_root = Path(mtp_search_root).resolve(strict = True) + if search_root in {gguf_parent, gguf_parent.parent}: + allowed_roots.add(search_root) + return companion_parent.parent in allowed_roots @dataclass(frozen = True) From 9beb299862e8a465594b99786052acd5c268ebdc Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:56:57 -0700 Subject: [PATCH 05/14] Studio: use scalable settings font token --- studio/frontend/src/features/chat/chat-settings-sheet.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index e2303d927e..a773db63b6 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -979,7 +979,7 @@ export function ChatSettingsPanel({ Delete -

+

Saving a preset also stores current load settings (context length, KV cache dtype, speculative decoding, GPU layers). {currentLoadSummary ? ( From 83d496ea48caa7897d12ea05362f49f19e7572c9 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:07:48 -0700 Subject: [PATCH 06/14] Studio: align local MTP source resolution --- studio/backend/models/inference.py | 6 ++++ studio/backend/routes/inference.py | 11 +++++++ .../tests/test_mtp_drafter_companion.py | 32 +++++++++++++++++++ .../tests/test_native_gguf_companion.py | 19 +++++++++++ studio/backend/utils/models/model_config.py | 16 ++++------ .../src/features/chat/api/chat-adapter.ts | 3 ++ .../src/features/chat/chat-settings-sheet.tsx | 6 +++- .../chat/hooks/use-chat-model-runtime.ts | 3 ++ .../lib/apply-inference-status-to-store.ts | 1 + .../src/features/chat/shared-composer.tsx | 1 + .../chat/stores/chat-runtime-store.ts | 4 +++ .../frontend/src/features/chat/types/api.ts | 2 ++ tests/studio/test_model_picker_contracts.py | 30 +++++++++++++++++ 13 files changed, 123 insertions(+), 11 deletions(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index 1758efe515..5aca8e3996 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -394,6 +394,9 @@ class LoadResponse(BaseModel): is_vision: bool = Field(False, description = "Whether model is a vision model") is_lora: bool = Field(False, description = "Whether model is a LoRA adapter") is_gguf: bool = Field(False, description = "Whether model is a GGUF model (llama.cpp)") + is_local_model: bool = Field( + False, description = "Whether the loaded model came from a local filesystem path" + ) is_diffusion: bool = Field( False, description = "Whether model is a block-diffusion model (DiffusionGemma)" ) @@ -558,6 +561,9 @@ class InferenceStatusResponse(BaseModel): ) is_vision: bool = Field(False, description = "Whether the active model is a vision model") is_gguf: bool = Field(False, description = "Whether the active model is a GGUF model (llama.cpp)") + is_local_model: bool = Field( + False, description = "Whether the active model came from a local filesystem path" + ) is_diffusion: bool = Field( False, description = "Whether the active model is a block-diffusion model (DiffusionGemma)" ) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index f4b8a94af5..91a7022398 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1018,6 +1018,7 @@ try: ) from core.inference.tensor_fallback import load_with_tensor_fallback from utils.models import ModelConfig + from utils.paths import is_local_path from utils.inference import load_inference_config from utils.models.model_config import ( _local_gguf_companion_search_root, @@ -1058,6 +1059,7 @@ except ImportError: ) from core.inference.tensor_fallback import load_with_tensor_fallback from utils.models import ModelConfig + from utils.paths import is_local_path from utils.inference import load_inference_config from utils.models.model_config import ( _local_gguf_companion_search_root, @@ -4420,6 +4422,8 @@ async def _load_model_impl( is_vision = llama_backend._is_vision, is_lora = False, is_gguf = True, + is_local_model = native_grant_backed + or is_local_path(llama_backend.model_identifier), is_diffusion = llama_backend.is_diffusion, is_audio = _gguf_is_audio, audio_type = _gguf_audio, @@ -4478,6 +4482,7 @@ async def _load_model_impl( is_vision = _model_info.get("is_vision", False), is_lora = _model_info.get("is_lora", False), is_gguf = False, + is_local_model = native_grant_backed or is_local_path(backend.active_model_name), is_audio = _model_info.get("is_audio", False), audio_type = _model_info.get("audio_type"), has_audio_input = _model_info.get("has_audio_input", False), @@ -4798,6 +4803,7 @@ async def _load_model_impl( is_vision = llama_backend.is_vision, is_lora = False, is_gguf = True, + is_local_model = config.is_local, is_diffusion = llama_backend.is_diffusion, is_audio = _gguf_is_audio, audio_type = _gguf_audio, @@ -4938,6 +4944,7 @@ async def _load_model_impl( is_vision = config.is_vision, is_lora = config.is_lora, is_gguf = False, + is_local_model = config.is_local, is_audio = config.is_audio, audio_type = config.audio_type, has_audio_input = config.has_audio_input, @@ -5918,6 +5925,7 @@ async def get_status(current_subject: str = Depends(get_current_subject)): model_identifier = None if _native_grant_backed else _model_id, is_vision = llama_backend.is_vision, is_gguf = True, + is_local_model = _native_grant_backed or bool(_model_id and is_local_path(_model_id)), is_diffusion = llama_backend.is_diffusion, gguf_variant = llama_backend.hf_variant, is_audio = getattr(llama_backend, "_is_audio", False), @@ -5989,6 +5997,9 @@ async def get_status(current_subject: str = Depends(get_current_subject)): model_identifier = backend.active_model_name, is_vision = is_vision, is_gguf = False, + is_local_model = bool( + backend.active_model_name and is_local_path(backend.active_model_name) + ), is_audio = is_audio, audio_type = audio_type, has_audio_input = has_audio_input, diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 5c1e7fb035..a3ac2e5071 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -29,7 +29,9 @@ from hub.utils.gguf_plan import ( preferred_mtp_sibling, ) from utils.models.model_config import ( + ModelConfig, _is_mtp_drafter, + _local_gguf_companion_search_root, detect_gguf_model, detect_mtp_file, extract_model_size_b, @@ -212,6 +214,36 @@ def test_detect_mtp_file_search_root(tmp_path): assert found is not None and found.endswith("mtp-gemma-4-12b-it.gguf") +def test_quant_directory_selection_finds_repo_root_mtp(tmp_path): + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + mtp_dir = tmp_path / "MTP" + mtp_dir.mkdir() + drafter = mtp_dir / "mtp-gemma-4-E4B-it-Q4_0.gguf" + drafter.write_bytes(b"x") + + search_root = _local_gguf_companion_search_root(str(quant_dir), str(weight)) + assert Path(search_root).resolve() == tmp_path.resolve() + config = ModelConfig.from_identifier(str(quant_dir)) + assert config.is_local + assert config.gguf_file == str(weight.resolve()) + assert config.gguf_mtp_file == str(drafter.resolve()) + + +def test_bare_relative_gguf_directory_is_local_source(tmp_path, monkeypatch): + model_dir = tmp_path / "outputs" / "gemma" + model_dir.mkdir(parents = True) + weight = model_dir / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + monkeypatch.chdir(tmp_path) + + config = ModelConfig.from_identifier("outputs/gemma") + assert config.is_local + assert config.gguf_file == str(weight.resolve()) + + def test_detect_mtp_file_falls_back_to_new_scheme_subdir(tmp_path): weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" weight.write_bytes(b"x") diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index 9ed260aa90..33e3c70dbb 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -99,6 +99,25 @@ def test_reload_dedup_finds_repo_root_mtp_companion(tmp_path, monkeypatch): assert _request_matches_loaded_settings(request, backend) +def test_reload_dedup_matches_quant_directory_selection(tmp_path, monkeypatch): + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model.gguf" + companion.write_bytes(b"draft") + + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._gguf_path = str(weight) + backend._mtp_draft_path = str(companion) + + request = LoadRequest(model_path = str(quant_dir), gguf_variant = "Q4_0") + assert _request_matches_loaded_settings(request, backend) + + def test_native_vision_companion_rejects_mtp_directory(tmp_path): weight, companion = _write_pair(tmp_path, "MTP") with pytest.raises(HTTPException, match = "must live next to"): diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 0d66515182..34c0acda64 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1765,13 +1765,6 @@ def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str selected = Path(selected_path) gguf_path = Path(gguf_file) - if selected.suffix.lower() != ".gguf": - return selected_path - - gguf_dir = gguf_path.parent - if not gguf_dir.name: - return str(gguf_dir) - quant_dir_re = ( r"(UD-)?(" r"MXFP[0-9]+(?:_[A-Z0-9]+)*" @@ -1783,9 +1776,12 @@ def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str r"|BF16|F16|F32" r")" ) - if re.fullmatch(quant_dir_re, gguf_dir.name, re.IGNORECASE): - return str(gguf_dir.parent) - return str(gguf_dir) + search_dir = gguf_path.parent if selected.suffix.lower() == ".gguf" else selected + if not search_dir.name: + return str(search_dir) + if re.fullmatch(quant_dir_re, search_dir.name, re.IGNORECASE): + return str(search_dir.parent) + return str(search_dir) def _iter_hf_cache_snapshots(repo_id: str, cache_dir: Optional[str | Path] = None): diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index d4861e8a3a..822b8627be 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -1704,6 +1704,7 @@ async function autoLoadSmallestModel(): Promise<{ customContextLength: config.customContextLength, loadedIsMultimodal: isMultimodalResponse(loadResp), loadedIsDiffusion: loadResp.is_diffusion ?? false, + activeModelIsLocal: loadResp.is_local_model ?? false, ...resolveLoadedSpeculativeSettings(loadResp), }); } else { @@ -1729,6 +1730,7 @@ async function autoLoadSmallestModel(): Promise<{ ...resolveLoadedSpeculativeSettings(loadResp), loadedIsMultimodal: isMultimodalResponse(loadResp), loadedIsDiffusion: loadResp.is_diffusion ?? false, + activeModelIsLocal: loadResp.is_local_model ?? false, }); } if (!(loadResp.is_lora ?? false)) { @@ -2007,6 +2009,7 @@ async function autoLoadSmallestModel(): Promise<{ defaultChatTemplate: loadResp.chat_template ?? null, chatTemplateOverride: null, loadedIsMultimodal: isMultimodalResponse(loadResp), + activeModelIsLocal: loadResp.is_local_model ?? false, ...resolveLoadedSpeculativeSettings(loadResp), }); recordLastLocalModelLoad({ diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index a773db63b6..c9d59d1b14 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -384,6 +384,9 @@ export function ChatSettingsPanel({ const activeNativePathToken = useChatRuntimeStore( (s) => s.activeNativePathToken, ); + const activeModelIsLocal = useChatRuntimeStore( + (s) => s.activeModelIsLocal, + ); const ggufContextLength = useChatRuntimeStore((s) => s.ggufContextLength); // Direct-file / custom-folder GGUFs load without a variant label but still // report a GGUF context, so detect them via the context and the checkpoint @@ -395,7 +398,8 @@ export function ChatSettingsPanel({ (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false); const isLocalGguf = isGguf && - (activeNativePathToken != null || + (activeModelIsLocal || + activeNativePathToken != null || isLocalModelPath(currentCheckpoint ?? "") || (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false)); const ggufMaxContextLength = useChatRuntimeStore( diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 76a310ac33..000eda6b62 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -319,6 +319,7 @@ async function syncInferenceStatusToStore(options?: { modelRequiresTrustRemoteCode: false, loadedIsMultimodal: false, loadedIsDiffusion: false, + activeModelIsLocal: false, }); } } catch (error) { @@ -1004,6 +1005,7 @@ export function useChatModelRuntime() { loadedChatTemplateOverride: effectiveChatTemplateOverride, loadedIsMultimodal: isMultimodalResponse(loadResponse), loadedIsDiffusion: loadResponse.is_diffusion ?? false, + activeModelIsLocal: loadResponse.is_local_model ?? false, activeNativePathToken: nativePathToken ?? null, activeNativePathExpiresAtMs: nativePathToken ? nativePathExpiresAtMs @@ -1113,6 +1115,7 @@ export function useChatModelRuntime() { rollbackResponse.speculative_type, ); useChatRuntimeStore.setState({ + activeModelIsLocal: rollbackResponse.is_local_model ?? false, activeNativePathToken: previousActiveNativePathToken ?? null, // Restore the previous token's lease together with the token so a // rollback never pairs restored token A with failed load B's expiry. diff --git a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts index f85ff3246b..31ccd3721c 100644 --- a/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts +++ b/studio/frontend/src/features/chat/lib/apply-inference-status-to-store.ts @@ -289,6 +289,7 @@ export function applyActiveModelStatusToStore( defaultChatTemplate: nextDefaultChatTemplate, loadedIsMultimodal: isMultimodalResponse(status), loadedIsDiffusion: status.is_diffusion ?? false, + activeModelIsLocal: status.is_local_model ?? false, specFallbackReason: status.spec_fallback_reason ?? null, // The spec / KV seeds share the GPU-fields reseed mechanism below: a // non-GGUF status leaves their loaded baselines null, so the "unseeded" diff --git a/studio/frontend/src/features/chat/shared-composer.tsx b/studio/frontend/src/features/chat/shared-composer.tsx index 3f89f70ef1..28d9e6673b 100644 --- a/studio/frontend/src/features/chat/shared-composer.tsx +++ b/studio/frontend/src/features/chat/shared-composer.tsx @@ -1226,6 +1226,7 @@ export function SharedComposer({ // GPU fields on every load path so the gate can't read stale. loadedIsDiffusion: resp.is_diffusion ?? false, loadedIsMultimodal: isMultimodalResponse(resp), + activeModelIsLocal: resp.is_local_model ?? false, // Record the context this pane loaded with (like the single-model path) // so when it becomes the active model, the UI and later reload/save use // its context, not the previous/default one. diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 42359b8f7f..9409945072 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -735,6 +735,8 @@ type ChatRuntimeStore = { // lets the attach gates flag a failed load vs "no model picked". lastModelLoadError: string | null; activeGgufVariant: string | null; + /** Whether the backend loaded the active model from a filesystem path. */ + activeModelIsLocal: boolean; ggufContextLength: number | null; ggufMaxContextLength: number | null; ggufNativeContextLength: number | null; @@ -1268,6 +1270,7 @@ export const useChatRuntimeStore = create((set, get) => ({ modelsError: null, lastModelLoadError: null, activeGgufVariant: null, + activeModelIsLocal: false, ggufContextLength: null, ggufMaxContextLength: null, ggufNativeContextLength: null, @@ -1555,6 +1558,7 @@ export const useChatRuntimeStore = create((set, get) => ({ checkpoint: "", }, activeGgufVariant: null, + activeModelIsLocal: false, activeNativePathToken: null, activeNativePathExpiresAtMs: null, ggufContextLength: null, diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index 6c3e919efe..e5b4239947 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -152,6 +152,7 @@ export interface LoadModelResponse { is_vision: boolean; is_lora: boolean; is_gguf?: boolean; + is_local_model?: boolean; is_diffusion?: boolean; is_audio?: boolean; audio_type?: string | null; @@ -203,6 +204,7 @@ export interface InferenceStatusResponse { model_identifier?: string | null; is_vision: boolean; is_gguf?: boolean; + is_local_model?: boolean; is_diffusion?: boolean; gguf_variant?: string | null; is_audio?: boolean; diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 1dc9761c12..c946a5de4a 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -26,6 +26,12 @@ def _read(rel: str) -> str: return path.read_text() +def _read_backend(rel: str) -> str: + path = WORKDIR / "studio" / "backend" / rel + assert path.exists(), f"missing backend source file: {path}" + return path.read_text() + + def test_models_api_sends_token_via_header_not_query(): """getModelConfig / checkVisionModel / checkEmbeddingModel must pass the HF token through hubTokenHeader, never as a ?hf_token= query param (which leaks @@ -326,10 +332,34 @@ def test_local_mtp_warning_covers_path_and_native_gguf_sources(): assert local assert "isGguf &&" in local.group(0) assert "activeNativePathToken" in local.group(0) + assert "activeModelIsLocal" in local.group(0) assert "isLocalModelPath" in local.group(0) assert "isLocalGguf" in src.split('specFallbackReason === "drafter_not_found"', 1)[1] +def test_local_mtp_warning_uses_backend_source_metadata(): + types = _read("features/chat/types/api.ts") + assert types.count("is_local_model?: boolean") >= 2 + + status = _read("features/chat/lib/apply-inference-status-to-store.ts") + assert "activeModelIsLocal: status.is_local_model ?? false" in status + + runtime = _read("features/chat/stores/chat-runtime-store.ts") + assert "activeModelIsLocal: boolean" in runtime + assert runtime.count("activeModelIsLocal: false") >= 2 + + load = _read("features/chat/hooks/use-chat-model-runtime.ts") + assert "activeModelIsLocal: loadResponse.is_local_model ?? false" in load + + models = _read_backend("models/inference.py") + assert models.count("is_local_model: bool = Field(") >= 2 + + route = _read_backend("routes/inference.py") + assert route.count("is_local_model = config.is_local") >= 2 + assert "is_local_model = _native_grant_backed" in route + assert "backend.active_model_name and is_local_path(backend.active_model_name)" in route + + def test_fixed_layer_gguf_pins_displayed_context(): """An already-loaded auto-fit GGUF saved with Manual fixed GPU layers must pin the shown context, so a later fresh load keeps the fitted placement From 6690d1b009791b304207228f5f7bd16aed89b688 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 01:32:55 -0700 Subject: [PATCH 07/14] Studio: widen MTP drafter pairing and recover subdir copies Follow-up fixes to the local MTP subdirectory discovery in this PR. - pair drafters using the module's full quant vocabulary instead of a local q_/bf16/f16 subset, so K, IQ, UD and MXFP copies match - strip the shard suffix before the anchored quant strip, so sharded drafters pair instead of falling through - order subdirectory candidates by real file size, with precision as a tie break; the previous fixed list ranked unknown quants behind BF16, so a small K-quant lost to a much larger BF16 copy - fall back to the MTP/ copy on native loads when the preferred root drafter sits outside the granted directory, instead of dropping the drafter and losing speculative decoding entirely - require a published drafter name inside MTP/ (mtp- or -MTP). _is_mtp_drafter accepts everything in that directory by design for variant-menu exclusion, which is too broad to include on: a weight copy placed there was launching as --model-draft - stop clearing activeModelIsLocal when the status poll reports no active model. specFallbackReason survives that branch, so clearing local-ness alone flipped a local model's warning back to the download-failed text --- studio/backend/routes/inference.py | 47 +++++++++--- studio/backend/utils/models/model_config.py | 75 +++++++++++++------ .../chat/hooks/use-chat-model-runtime.ts | 4 +- 3 files changed, 89 insertions(+), 37 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 91a7022398..370d76cef3 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4669,20 +4669,43 @@ async def _load_model_impl( if config.gguf_mtp_file: # The drafter is optional (unlike mmproj for a vision # model): drop it rather than fail the load. - try: - mtp_search_root = _local_gguf_companion_search_root( - config.gguf_file, config.gguf_file - ) - _validate_native_gguf_companion( - config.gguf_mtp_file, + mtp_search_root = _local_gguf_companion_search_root( + config.gguf_file, config.gguf_file + ) + + def _mtp_allowed(candidate: str) -> bool: + try: + _validate_native_gguf_companion( + candidate, + config.gguf_file, + "MTP drafter", + allow_mtp_subdir = True, + mtp_search_root = mtp_search_root, + ) + return True + except HTTPException as exc: + logger.warning( + "Dropping MTP drafter for native load: %s", exc.detail + ) + return False + + if not _mtp_allowed(config.gguf_mtp_file): + # The preferred root drafter is out of bounds for a + # grant on a quant subdir, but its MTP/ copy may + # not be. Use that before dropping MTP entirely. + fallback = detect_mtp_file( config.gguf_file, - "MTP drafter", - allow_mtp_subdir = True, - mtp_search_root = mtp_search_root, + search_root = mtp_search_root, + skip_root = True, ) - except HTTPException as exc: - logger.warning("Dropping MTP drafter for native load: %s", exc.detail) - config.gguf_mtp_file = None + if fallback and _mtp_allowed(fallback): + logger.info( + "Using MTP subdirectory drafter for native load: %s", + fallback, + ) + config.gguf_mtp_file = fallback + else: + config.gguf_mtp_file = None _source_load_kwargs = dict( gguf_path = config.gguf_file, mmproj_path = config.gguf_mmproj_file, diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 34c0acda64..566abe9d3b 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1441,7 +1441,11 @@ def detect_mmproj_file(path: str, search_root: Optional[str] = None) -> Optional return str(best[1]) -def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[str]: +def detect_mtp_file( + path: str, + search_root: Optional[str] = None, + skip_root: bool = False, +) -> Optional[str]: """Find the separate MTP drafter (``mtp-*.gguf``) for a local GGUF model. The drafter that pairs with the main weights sits at the repo/snapshot @@ -1457,15 +1461,22 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st ``mtp-gemma-4-12B-it.gguf`` next to ``gemma-4-12B-it-qat-Q4_0.gguf``). If the root drafter is absent, also accept its precision copy under the repository's ``MTP/`` directory. An unmatched drafter is skipped. + + ``skip_root`` scans only ``MTP/``, for callers that must discard an + out-of-bounds root drafter and still want the subdir copy (native loads). """ def _pairing_stem(name: str) -> str: stem = Path(name).stem.lower() if stem.startswith("mtp-"): stem = stem[len("mtp-") :] + # Shard suffix sits outside the quant token, so strip it first or the + # anchored strip below cannot match. + stem = re.sub(r"-[0-9]+-of-[0-9]+$", "", stem) if stem.endswith("-mtp"): stem = stem[: -len("-mtp")] - return re.sub(r"-(?:q[0-9]+_[0-9]+|bf16|f16)$", "", stem) + # Full quant vocabulary, not a subset: K/IQ/UD/MXFP drafters pair too. + return re.sub(rf"-(?:{_GGUF_KNOWN_QUANT_RE.pattern})$", "", stem, flags = re.IGNORECASE) def _matches_weight(candidate: Path) -> bool: if weight_name is None: @@ -1477,17 +1488,24 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st and (len(weight_name) == len(stem) or not weight_name[len(stem)].isalnum()) ) - def _precision_rank(candidate: Path) -> tuple[int, str]: + def _smallest_first(candidate: Path) -> tuple[int, int, str]: + # Cheapest compatible copy wins. Size first: a fixed precision list + # ranked unknown quants behind BF16, so a small K-quant lost to a far + # larger BF16. Precision breaks size ties, name keeps it stable. name = candidate.name.lower() + try: + size = candidate.stat().st_size + except OSError: + size = sys.maxsize if "-q4_0" in name: - rank = 0 + precision = 0 elif "-q8_0" in name: - rank = 1 + precision = 1 elif "-bf16" in name or "-f16" in name: - rank = 2 + precision = 2 else: - rank = 3 - return rank, name + precision = 3 + return size, precision, name p = Path(path) weight_name = p.name.lower() if p.suffix.lower() == ".gguf" else None @@ -1495,22 +1513,23 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st dirs = [start_dir] if search_root is not None: dirs.append(Path(search_root)) - for d in dirs: - try: - entries = sorted(d.iterdir()) - except OSError: - continue - for f in entries: - name = f.name.lower() - if not (name.startswith("mtp-") and name.endswith(".gguf")): - continue - if not _matches_weight(f): - continue + if not skip_root: + for d in dirs: try: - if f.is_file(): - return str(f.resolve()) + entries = sorted(d.iterdir()) except OSError: continue + for f in entries: + name = f.name.lower() + if not (name.startswith("mtp-") and name.endswith(".gguf")): + continue + if not _matches_weight(f): + continue + try: + if f.is_file(): + return str(f.resolve()) + except OSError: + continue subdir_candidates: list[Path] = [] for d in dirs: @@ -1533,8 +1552,16 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st except OSError: continue for f in entries: - rel = f"MTP/{f.name}" - if not _is_mtp_drafter(rel) or not _matches_weight(f): + # _is_mtp_drafter accepts everything under MTP/ by design (it + # excludes them from variant menus). Too broad to include on: + # a weight copy here would launch as --model-draft. Require a + # published drafter name: mtp- or -MTP. + lower = f.name.lower() + if not lower.endswith(".gguf"): + continue + if not (lower.startswith("mtp-") or Path(lower).stem.endswith("-mtp")): + continue + if not _matches_weight(f): continue try: if f.is_file(): @@ -1542,7 +1569,7 @@ def detect_mtp_file(path: str, search_root: Optional[str] = None) -> Optional[st except OSError: continue - for candidate in sorted(subdir_candidates, key = _precision_rank): + for candidate in sorted(subdir_candidates, key = _smallest_first): try: resolved = candidate.resolve() except OSError: diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 000eda6b62..d2d20396c7 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -315,11 +315,13 @@ async function syncInferenceStatusToStore(options?: { syncModelCapabilities(checkpointId, statusRes); } } else if (!statusRes.active_model && !isExternalSelectionActive) { + // specFallbackReason survives here, so clearing activeModelIsLocal + // alone would flip a local model's warning to "download failed". Every + // load path and clearCheckpoint set it, so leave it consistent. useChatRuntimeStore.setState({ modelRequiresTrustRemoteCode: false, loadedIsMultimodal: false, loadedIsDiffusion: false, - activeModelIsLocal: false, }); } } catch (error) { From 60757367ab558447b5511a215ea2c07a758e975b Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 05:51:58 -0700 Subject: [PATCH 08/14] Studio: fix sharded MTP picks and native fallback dedup Addresses the two review points that apply to the current revision. - collapse a split MTP/ drafter to its first shard before ranking, so size ordering cannot select a smaller trailing shard. llama-server takes shard 1 as the model path, matching _local_gguf_load_path - align the shard suffix pattern with _GGUF_SPLIT_FILE_RE - accept the MTP/ fallback during reload deduplication. A native load whose root drafter is out of bounds launches the subdir copy, so root-first detection never equalled the stored path and that layout restarted llama-server on every apply. A deleted drafter still forces a reload The two review points about the companion root during deduplication do not apply: for a quant named directory the load-time and dedup roots both resolve to the repository root. The divergence is limited to subdirectories whose names do not match the quant pattern, which predates this branch and is noted in the description. --- studio/backend/routes/inference.py | 14 ++++- .../tests/test_mtp_drafter_companion.py | 57 +++++++++++++++++++ .../tests/test_native_gguf_companion.py | 43 ++++++++++++++ studio/backend/utils/models/model_config.py | 8 ++- 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 370d76cef3..878a55bc92 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3356,7 +3356,19 @@ def _request_matches_loaded_settings( except OSError: return False if detected_resolved != stored_resolved: - return False + # A native load whose root drafter was out of bounds runs the + # MTP/ fallback instead, so root-first detection never equals + # what launched. Accept the subdir copy as current too, else + # that layout reloads on every apply. + fallback = detect_mtp_file( + llama_backend.gguf_path, search_root = companion_root, skip_root = True + ) + try: + fallback_resolved = Path(fallback).resolve() if fallback else None + except OSError: + return False + if stored_resolved is None or fallback_resolved != stored_resolved: + return False return True diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index a3ac2e5071..2ffeb62d85 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -632,3 +632,60 @@ def test_download_mtp_online_skips_cache_reuse(tmp_path, monkeypatch): b._download_companion_gguf = _fake_companion assert b._download_mtp(hf_repo = "unsloth/gemma-4-E4B-it-qat-mobile-GGUF") is None assert reached.get("hit") is True + + +def test_detect_mtp_file_returns_first_shard_of_split_subdir_drafter(tmp_path): + """llama-server takes shard 1 as the model path, so a split MTP/ copy must + not resolve to whichever shard happens to be smallest.""" + weight = tmp_path / "model-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + first = sub / "mtp-model-Q4_0-00001-of-00002.gguf" + first.write_bytes(b"x" * 4096) + (sub / "mtp-model-Q4_0-00002-of-00002.gguf").write_bytes(b"x") + + assert detect_mtp_file(str(weight)) == str(first.resolve()) + + +def test_detect_mtp_file_skip_root_ignores_root_drafter(tmp_path): + """skip_root is how a native load recovers when the root drafter is out + of bounds for its grant.""" + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"x") + (tmp_path / "mtp-model.gguf").write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + subdir_copy = sub / "mtp-model-Q4_0.gguf" + subdir_copy.write_bytes(b"x") + + assert detect_mtp_file(str(weight), str(tmp_path)) == str( + (tmp_path / "mtp-model.gguf").resolve() + ) + assert detect_mtp_file(str(weight), str(tmp_path), skip_root = True) == str(subdir_copy.resolve()) + + +def test_detect_mtp_file_rejects_weight_copy_inside_mtp_dir(tmp_path): + """Everything under MTP/ counts as a drafter for menu exclusion, but only + a published drafter name may be launched as --model-draft.""" + weight = tmp_path / "gemma-4-E4B-it-qat-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + (sub / "gemma-4-E4B-it-qat-Q4_0.gguf").write_bytes(b"x") + + assert detect_mtp_file(str(weight)) is None + + +def test_detect_mtp_file_pairs_k_quant_subdir_drafter(tmp_path): + """Pairing must use the full quant vocabulary, not just Q_/BF16/F16.""" + weight = tmp_path / "gemma-4-12b-it-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + drafter = sub / "mtp-gemma-4-12b-it-UD-Q4_K_XL.gguf" + drafter.write_bytes(b"x") + + assert detect_mtp_file(str(weight)) == str(drafter.resolve()) diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index 33e3c70dbb..105f7b5e1a 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -194,3 +194,46 @@ def test_native_companion_rejects_missing_weight(tmp_path): def test_native_companion_none_is_noop(): _validate_native_gguf_companion(None, None, "MTP drafter") + + +def test_reload_dedup_accepts_native_subdir_fallback(tmp_path, monkeypatch): + """A native load whose root drafter was out of bounds launches the MTP/ + copy, so root-first detection never matches it. Dedup must still hold.""" + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + (tmp_path / "mtp-model.gguf").write_bytes(b"root drafter") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model-Q4_0.gguf" + companion.write_bytes(b"draft") + + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._gguf_path = str(weight) + backend._mtp_draft_path = str(companion) + + request = LoadRequest(model_path = str(weight)) + assert _request_matches_loaded_settings(request, backend) + + +def test_reload_dedup_still_reloads_when_drafter_disappears(tmp_path, monkeypatch): + """The fallback comparison must not mask a deleted drafter.""" + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model-Q4_0.gguf" + companion.write_bytes(b"draft") + + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._gguf_path = str(weight) + backend._mtp_draft_path = str(companion) + + companion.unlink() + request = LoadRequest(model_path = str(weight)) + assert not _request_matches_loaded_settings(request, backend) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 566abe9d3b..1a6036765a 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1472,7 +1472,7 @@ def detect_mtp_file( stem = stem[len("mtp-") :] # Shard suffix sits outside the quant token, so strip it first or the # anchored strip below cannot match. - stem = re.sub(r"-[0-9]+-of-[0-9]+$", "", stem) + stem = re.sub(r"-[0-9]{5}-of-[0-9]{5}$", "", stem) if stem.endswith("-mtp"): stem = stem[: -len("-mtp")] # Full quant vocabulary, not a subset: K/IQ/UD/MXFP drafters pair too. @@ -1565,11 +1565,13 @@ def detect_mtp_file( continue try: if f.is_file(): - subdir_candidates.append(f) + # llama-server takes shard 1 as the model path, so + # collapse a split copy to it before ranking. + subdir_candidates.append(_local_gguf_load_path(f)) except OSError: continue - for candidate in sorted(subdir_candidates, key = _smallest_first): + for candidate in sorted(dict.fromkeys(subdir_candidates), key = _smallest_first): try: resolved = candidate.resolve() except OSError: From fc90affb29ce3bf1e95e836781bff00688325326 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:20:24 -0700 Subject: [PATCH 09/14] Studio: keep sharded MTP snapshot paths and gate dedup on native loads Both review points on the previous revision were correct. - stop resolving a split MTP/ drafter to its blob target. Snapshot symlinks are how HF stores shards, and the blob has no sibling shard names, so --model-draft could not load. _local_gguf_load_path already preserves the snapshot path; the later resolve() was undoing it. Single file drafters still resolve as before - restrict the reload deduplication fallback to native loads. An ordinary local load can reach a root drafter added after the fact and must reload to pick it up; only a native load, whose root candidate is outside the lease, keeps running the MTP/ copy Tests cover the snapshot shard path and both deduplication routes. --- studio/backend/routes/inference.py | 8 ++++- .../tests/test_mtp_drafter_companion.py | 28 +++++++++++++++++ .../tests/test_native_gguf_companion.py | 31 ++++++++++++++++++- studio/backend/utils/models/model_config.py | 6 +++- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index f26561673b..2b98d06aff 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3215,6 +3215,7 @@ def _request_matches_loaded_settings( request: LoadRequest, llama_backend: LlamaCppBackend, effective_chat_template_override: Optional[str] = None, + native_grant_backed: bool = False, ) -> bool: """True iff every runtime setting on the request matches the loaded server. Caller has already checked model+variant+is_loaded. See #5401. @@ -3368,7 +3369,11 @@ def _request_matches_loaded_settings( # A native load whose root drafter was out of bounds runs the # MTP/ fallback instead, so root-first detection never equals # what launched. Accept the subdir copy as current too, else - # that layout reloads on every apply. + # that layout reloads on every apply. Native only: an ordinary + # load can reach the root drafter, so a newly added one must + # still reload. + if not native_grant_backed: + return False fallback = detect_mtp_file( llama_backend.gguf_path, search_root = companion_root, skip_root = True ) @@ -4419,6 +4424,7 @@ async def _load_model_impl( request, llama_backend, effective_chat_template_override, + native_grant_backed = native_grant_backed, ) # Skip if a prior audio probe failed -- let load_model retry. and getattr(llama_backend, "_audio_probed", True) diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 2ffeb62d85..15705753b4 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -689,3 +689,31 @@ def test_detect_mtp_file_pairs_k_quant_subdir_drafter(tmp_path): drafter.write_bytes(b"x") assert detect_mtp_file(str(weight)) == str(drafter.resolve()) + + +def test_detect_mtp_file_keeps_snapshot_path_for_sharded_subdir_drafter(tmp_path): + """A split copy stored as HF snapshot symlinks must launch from the + snapshot path: the blob target has no sibling shard names.""" + blobs = tmp_path / "blobs" + snapshot = tmp_path / "snapshots" / "abc" + sub = snapshot / "MTP" + blobs.mkdir(parents = True) + sub.mkdir(parents = True) + + (blobs / "sha_weight").write_bytes(b"w") + weight = snapshot / "model-Q4_0.gguf" + try: + weight.symlink_to(blobs / "sha_weight") + except OSError: + pytest.skip("symlinks unavailable") + + first = sub / "mtp-model-Q4_0-00001-of-00002.gguf" + second = sub / "mtp-model-Q4_0-00002-of-00002.gguf" + (blobs / "sha_1").write_bytes(b"d" * 4096) + (blobs / "sha_2").write_bytes(b"d") + first.symlink_to(blobs / "sha_1") + second.symlink_to(blobs / "sha_2") + + found = detect_mtp_file(str(weight), str(snapshot)) + assert found == str(first) + assert (Path(found).parent / second.name).exists() diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index 105f7b5e1a..24f0553580 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -215,7 +215,7 @@ def test_reload_dedup_accepts_native_subdir_fallback(tmp_path, monkeypatch): backend._mtp_draft_path = str(companion) request = LoadRequest(model_path = str(weight)) - assert _request_matches_loaded_settings(request, backend) + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) def test_reload_dedup_still_reloads_when_drafter_disappears(tmp_path, monkeypatch): @@ -237,3 +237,32 @@ def test_reload_dedup_still_reloads_when_drafter_disappears(tmp_path, monkeypatc companion.unlink() request = LoadRequest(model_path = str(weight)) assert not _request_matches_loaded_settings(request, backend) + + +def test_reload_dedup_reloads_for_ordinary_load_when_root_drafter_appears(tmp_path, monkeypatch): + """The native fallback exception must not swallow a newly added root + drafter on an ordinary local load, which can reach it.""" + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model-Q4_0.gguf" + companion.write_bytes(b"draft") + + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._gguf_path = str(weight) + backend._mtp_draft_path = str(companion) + + request = LoadRequest(model_path = str(weight)) + # No root drafter yet: both routes dedupe. + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = False) + + (tmp_path / "mtp-model.gguf").write_bytes(b"root drafter") + # Native cannot reach the root drafter, so the subdir copy stays current. + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) + # An ordinary load would pick the root drafter, so it must reload. + assert not _request_matches_loaded_settings(request, backend, None, native_grant_backed = False) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 1a6036765a..8f1384df27 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1573,7 +1573,11 @@ def detect_mtp_file( for candidate in sorted(dict.fromkeys(subdir_candidates), key = _smallest_first): try: - resolved = candidate.resolve() + # A split copy keeps its snapshot path: resolving to the blob + # drops the sibling shard names llama-server needs to find. + resolved = ( + candidate if _GGUF_SPLIT_FILE_RE.match(candidate.name) else candidate.resolve() + ) except OSError: continue logger.info(f"Detected MTP subdirectory drafter: {resolved}") From e9784fd2123fc890562eaee02b1affbd3780724b Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 18:34:57 -0700 Subject: [PATCH 10/14] Studio: fix shard handling on the root branch and native dedup comparison All four review points on the previous revision were correct. - share one launch-path helper between the root and MTP/ branches. The root branch still resolved to the blob, so a sharded snapshot drafter lost its sibling shard names. It began matching once _pairing_stem learned shard suffixes, so the two branches had drifted - strip the shard suffix before the -mtp name check. An old-scheme split copy is -Q8_0-MTP-00001-of-00002.gguf, whose stem does not end in -mtp, so every shard was discarded - mirror the load path's admissibility choice during reload dedup for native loads instead of special casing a stored None. With a root drafter outside the grant and no MTP/ copy the load stores no drafter, and dedup compared that None against the root file and reloaded every time - drop activeNativePathToken from the local GGUF classification. Status reconciliation keeps the token across a switch to a remote GGUF because no replacement token exists, so a stale token labelled that remote model local and showed placement guidance instead of the download recovery text. activeModelIsLocal is the backend's own classification and already covers native picks The model picker contract now asserts the token is absent rather than present, since it pinned the stale-token behaviour. --- studio/backend/routes/inference.py | 58 +++++++++++++------ .../tests/test_mtp_drafter_companion.py | 40 +++++++++++++ .../tests/test_native_gguf_companion.py | 20 +++++++ studio/backend/utils/models/model_config.py | 28 +++++---- .../src/features/chat/chat-settings-sheet.tsx | 8 +-- tests/studio/test_model_picker_contracts.py | 4 +- 6 files changed, 126 insertions(+), 32 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 2b98d06aff..1944bc9dd4 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3138,6 +3138,27 @@ def _validate_native_gguf_companion( ) from exc +def _native_gguf_companion_usable( + companion_path: str | None, + gguf_path: str | None, + *, + mtp_search_root: str | Path | None = None, +) -> bool: + """Whether a native load would accept this MTP drafter. Same rules as + _validate_native_gguf_companion, as a predicate for reload dedup.""" + try: + _validate_native_gguf_companion( + companion_path, + gguf_path, + "MTP drafter", + allow_mtp_subdir = True, + mtp_search_root = mtp_search_root, + ) + except HTTPException: + return False + return True + + def _normalise_settings_str(value: Optional[str]) -> Optional[str]: """Lowercase + strip a settings string, mapping blank/None to None.""" if value is None: @@ -3359,6 +3380,25 @@ def _request_matches_loaded_settings( llama_backend.gguf_path, llama_backend.gguf_path ) detected = detect_mtp_file(llama_backend.gguf_path, search_root = companion_root) + if native_grant_backed: + # Mirror the load path's choice, or the comparison is against a + # drafter that never launched. A native grant cannot reach a + # root drafter outside it, so the load falls back to the MTP/ + # copy and, failing that, to no drafter at all. An ordinary + # load reaches the root drafter, so it keeps root-first + # detection and reloads when one appears. + if detected and not _native_gguf_companion_usable( + detected, llama_backend.gguf_path, mtp_search_root = companion_root + ): + detected = detect_mtp_file( + llama_backend.gguf_path, + search_root = companion_root, + skip_root = True, + ) + if detected and not _native_gguf_companion_usable( + detected, llama_backend.gguf_path, mtp_search_root = companion_root + ): + detected = None stored = llama_backend.mtp_draft_path try: detected_resolved = Path(detected).resolve() if detected else None @@ -3366,23 +3406,7 @@ def _request_matches_loaded_settings( except OSError: return False if detected_resolved != stored_resolved: - # A native load whose root drafter was out of bounds runs the - # MTP/ fallback instead, so root-first detection never equals - # what launched. Accept the subdir copy as current too, else - # that layout reloads on every apply. Native only: an ordinary - # load can reach the root drafter, so a newly added one must - # still reload. - if not native_grant_backed: - return False - fallback = detect_mtp_file( - llama_backend.gguf_path, search_root = companion_root, skip_root = True - ) - try: - fallback_resolved = Path(fallback).resolve() if fallback else None - except OSError: - return False - if stored_resolved is None or fallback_resolved != stored_resolved: - return False + return False return True diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 15705753b4..98a233ac2b 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -717,3 +717,43 @@ def test_detect_mtp_file_keeps_snapshot_path_for_sharded_subdir_drafter(tmp_path found = detect_mtp_file(str(weight), str(snapshot)) assert found == str(first) assert (Path(found).parent / second.name).exists() + + +def test_detect_mtp_file_pairs_sharded_old_scheme_subdir_drafter(tmp_path): + """An old-scheme split copy is -Q8_0-MTP-00001-of-00002.gguf, whose + stem does not end in -mtp until the shard suffix comes off.""" + weight = tmp_path / "model-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + first = sub / "model-Q8_0-MTP-00001-of-00002.gguf" + first.write_bytes(b"x" * 4096) + (sub / "model-Q8_0-MTP-00002-of-00002.gguf").write_bytes(b"x") + + assert detect_mtp_file(str(weight)) == str(first) + + +def test_detect_mtp_file_keeps_snapshot_path_for_sharded_root_drafter(tmp_path): + """The root branch needs the same shard handling as the MTP/ branch.""" + blobs = tmp_path / "blobs" + snapshot = tmp_path / "snapshots" / "abc" + blobs.mkdir(parents = True) + snapshot.mkdir(parents = True) + + (blobs / "sha_weight").write_bytes(b"w") + weight = snapshot / "model-Q4_0.gguf" + try: + weight.symlink_to(blobs / "sha_weight") + except OSError: + pytest.skip("symlinks unavailable") + + first = snapshot / "mtp-model-Q4_0-00001-of-00002.gguf" + second = snapshot / "mtp-model-Q4_0-00002-of-00002.gguf" + (blobs / "sha_1").write_bytes(b"d" * 4096) + (blobs / "sha_2").write_bytes(b"d") + first.symlink_to(blobs / "sha_1") + second.symlink_to(blobs / "sha_2") + + found = detect_mtp_file(str(weight), str(snapshot)) + assert found == str(first) + assert (Path(found).parent / second.name).exists() diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index 24f0553580..ab1ba4f7fe 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -266,3 +266,23 @@ def test_reload_dedup_reloads_for_ordinary_load_when_root_drafter_appears(tmp_pa assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) # An ordinary load would pick the root drafter, so it must reload. assert not _request_matches_loaded_settings(request, backend, None, native_grant_backed = False) + + +def test_reload_dedup_native_load_with_no_admissible_drafter(tmp_path, monkeypatch): + """Root drafter out of the grant and no MTP/ copy: the load stores no + drafter, so dedup must compare against None rather than the root file.""" + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + (tmp_path / "mtp-model.gguf").write_bytes(b"root drafter") + + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._gguf_path = str(weight) + backend._mtp_draft_path = None + + request = LoadRequest(model_path = str(weight)) + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) + # An ordinary load would launch the root drafter, so it must reload. + assert not _request_matches_loaded_settings(request, backend, None, native_grant_backed = False) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 8f1384df27..6e49294e3f 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1478,6 +1478,15 @@ def detect_mtp_file( # Full quant vocabulary, not a subset: K/IQ/UD/MXFP drafters pair too. return re.sub(rf"-(?:{_GGUF_KNOWN_QUANT_RE.pattern})$", "", stem, flags = re.IGNORECASE) + def _drafter_launch_path(candidate: Path) -> str: + # llama-server takes shard 1 as the model path, and a split copy must + # stay on its snapshot path: the blob target has no sibling shard + # names. Single-file drafters still resolve, as callers expect. + loadable = _local_gguf_load_path(candidate) + if _GGUF_SPLIT_FILE_RE.match(loadable.name): + return str(loadable) + return str(loadable.resolve()) + def _matches_weight(candidate: Path) -> bool: if weight_name is None: return True @@ -1527,7 +1536,7 @@ def detect_mtp_file( continue try: if f.is_file(): - return str(f.resolve()) + return _drafter_launch_path(f) except OSError: continue @@ -1559,29 +1568,28 @@ def detect_mtp_file( lower = f.name.lower() if not lower.endswith(".gguf"): continue - if not (lower.startswith("mtp-") or Path(lower).stem.endswith("-mtp")): + # Drop the shard suffix first: an old-scheme split copy is + # named -Q8_0-MTP-00001-of-00002.gguf, whose stem does + # not end in -mtp. + stem = re.sub(r"-[0-9]{5}-of-[0-9]{5}$", "", Path(lower).stem) + if not (lower.startswith("mtp-") or stem.endswith("-mtp")): continue if not _matches_weight(f): continue try: if f.is_file(): - # llama-server takes shard 1 as the model path, so - # collapse a split copy to it before ranking. + # Collapse a split copy to shard 1 before ranking. subdir_candidates.append(_local_gguf_load_path(f)) except OSError: continue for candidate in sorted(dict.fromkeys(subdir_candidates), key = _smallest_first): try: - # A split copy keeps its snapshot path: resolving to the blob - # drops the sibling shard names llama-server needs to find. - resolved = ( - candidate if _GGUF_SPLIT_FILE_RE.match(candidate.name) else candidate.resolve() - ) + resolved = _drafter_launch_path(candidate) except OSError: continue logger.info(f"Detected MTP subdirectory drafter: {resolved}") - return str(resolved) + return resolved return None diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index c9d59d1b14..d0a525ca18 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -381,9 +381,6 @@ export function ChatSettingsPanel({ const isMobile = useIsMobile(); const isLoadedGguf = useChatRuntimeStore((s) => s.activeGgufVariant) != null; const currentCheckpoint = params.checkpoint; - const activeNativePathToken = useChatRuntimeStore( - (s) => s.activeNativePathToken, - ); const activeModelIsLocal = useChatRuntimeStore( (s) => s.activeModelIsLocal, ); @@ -396,10 +393,13 @@ export function ChatSettingsPanel({ isLoadedGguf || ggufContextLength != null || (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false); + // activeModelIsLocal is the backend's own classification and covers native + // picks. activeNativePathToken must not be used here: status reconciliation + // keeps it across a switch to a remote GGUF (no replacement token exists), + // so a stale token would label that remote model local. const isLocalGguf = isGguf && (activeModelIsLocal || - activeNativePathToken != null || isLocalModelPath(currentCheckpoint ?? "") || (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false)); const ggufMaxContextLength = useChatRuntimeStore( diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 50464d31b9..c547f4927b 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -349,9 +349,11 @@ def test_local_mtp_warning_covers_path_and_native_gguf_sources(): local = re.search(r"const isLocalGguf =.*?;", src, re.S) assert local assert "isGguf &&" in local.group(0) - assert "activeNativePathToken" in local.group(0) assert "activeModelIsLocal" in local.group(0) assert "isLocalModelPath" in local.group(0) + # A native token outlives a switch to a remote GGUF, so it must not + # classify the model here; activeModelIsLocal already covers native picks. + assert "activeNativePathToken" not in local.group(0) assert "isLocalGguf" in src.split('specFallbackReason === "drafter_not_found"', 1)[1] From 2d1e2cc180aab51ffa13e204a20b196d7376fd10 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:06:51 -0700 Subject: [PATCH 11/14] Studio: complete shard handling and persist local provenance All five review points on the previous revision were correct. - pair bpw-qualified drafters such as mtp-model-IQ4_XS-3.53bpw.gguf. The anchored quant strip left the modifier behind, so the name never matched even though _extract_quant_label supports these filenames - skip an incomplete split candidate instead of launching shard 1 of a set that llama-server cannot start, which also let it outrank a complete copy and disable MTP outright - rank a split copy by its total shard size. Candidates collapse to shard 1, so a 90+90 split was beating a 100 byte single file - validate every shard of a split drafter under a native lease. Sibling shards are opened implicitly by llama-server, so checking only the launch path let a later shard be a symlink out of the permitted directory - record provenance at load time and report that from status. Deleting or unmounting a bare relative model directory underneath a running server made is_local_path read the identifier as an org/model repo id and flipped a local model to remote colocated_split_shards is the one place that enumerates a shard set, so detection, ranking and native validation cannot disagree about it. The model picker contract now pins the provenance helper rather than the inline status expression it replaced. --- studio/backend/routes/inference.py | 66 ++++++++++++++---- .../tests/test_mtp_drafter_companion.py | 42 ++++++++++++ .../tests/test_native_gguf_companion.py | 48 +++++++++++++ studio/backend/utils/models/model_config.py | 67 +++++++++++++++++-- tests/studio/test_model_picker_contracts.py | 6 +- 5 files changed, 211 insertions(+), 18 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 1944bc9dd4..6f204b7e38 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1022,6 +1022,7 @@ try: from utils.inference import load_inference_config from utils.models.model_config import ( _local_gguf_companion_search_root, + colocated_split_shards, detect_mtp_file, load_model_defaults, ) @@ -1063,6 +1064,7 @@ except ImportError: from utils.inference import load_inference_config from utils.models.model_config import ( _local_gguf_companion_search_root, + colocated_split_shards, detect_mtp_file, load_model_defaults, ) @@ -3138,22 +3140,56 @@ def _validate_native_gguf_companion( ) from exc +def _loaded_is_local_model( + llama_backend: LlamaCppBackend, native_grant_backed: bool, model_id: str | None +) -> bool: + """Provenance of the running model, preferring what the load recorded. + + Falls back to the filesystem for a server started before the flag existed. + """ + if native_grant_backed: + return True + stored = getattr(llama_backend, "_is_local_model", None) + if stored is not None: + return bool(stored) + return bool(model_id and is_local_path(model_id)) + + +def _validate_native_mtp_drafter( + companion_path: str | None, + gguf_path: str | None, + *, + mtp_search_root: str | Path | None = None, +) -> None: + """Validate an MTP drafter for a native load, every shard of it. + + llama-server opens the sibling shards of a split drafter implicitly, so + checking only the launch path would let a later shard be a symlink out of + the permitted directory without ever facing the native rules. + """ + if not companion_path or not gguf_path: + return + shards, _ = colocated_split_shards(Path(companion_path)) + for shard in shards or [Path(companion_path)]: + _validate_native_gguf_companion( + str(shard), + gguf_path, + "MTP drafter", + allow_mtp_subdir = True, + mtp_search_root = mtp_search_root, + ) + + def _native_gguf_companion_usable( companion_path: str | None, gguf_path: str | None, *, mtp_search_root: str | Path | None = None, ) -> bool: - """Whether a native load would accept this MTP drafter. Same rules as - _validate_native_gguf_companion, as a predicate for reload dedup.""" + """Whether a native load would accept this MTP drafter, as a predicate for + reload dedup. Same rules, so the two cannot disagree.""" try: - _validate_native_gguf_companion( - companion_path, - gguf_path, - "MTP drafter", - allow_mtp_subdir = True, - mtp_search_root = mtp_search_root, - ) + _validate_native_mtp_drafter(companion_path, gguf_path, mtp_search_root = mtp_search_root) except HTTPException: return False return True @@ -4726,11 +4762,9 @@ async def _load_model_impl( def _mtp_allowed(candidate: str) -> bool: try: - _validate_native_gguf_companion( + _validate_native_mtp_drafter( candidate, config.gguf_file, - "MTP drafter", - allow_mtp_subdir = True, mtp_search_root = mtp_search_root, ) return True @@ -4865,6 +4899,10 @@ async def _load_model_impl( _gguf_is_audio = llama_backend._is_audio llama_backend._native_display_label = model_log_label if native_grant_backed else None llama_backend._native_grant_backed = bool(native_grant_backed) + # Provenance is a load-time fact. Re-deriving it per status poll + # would flip a local model to remote if its directory is deleted + # or unmounted underneath a still-running server. + llama_backend._is_local_model = bool(native_grant_backed or config.is_local) if _gguf_is_audio: logger.info(f"GGUF model detected as audio: audio_type={_gguf_audio}") @@ -5999,7 +6037,9 @@ async def get_status(current_subject: str = Depends(get_current_subject)): model_identifier = None if _native_grant_backed else _model_id, is_vision = llama_backend.is_vision, is_gguf = True, - is_local_model = _native_grant_backed or bool(_model_id and is_local_path(_model_id)), + is_local_model = _loaded_is_local_model( + llama_backend, _native_grant_backed, _model_id + ), is_diffusion = llama_backend.is_diffusion, gguf_variant = llama_backend.hf_variant, is_audio = getattr(llama_backend, "_is_audio", False), diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 98a233ac2b..bf1ef1a061 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -757,3 +757,45 @@ def test_detect_mtp_file_keeps_snapshot_path_for_sharded_root_drafter(tmp_path): found = detect_mtp_file(str(weight), str(snapshot)) assert found == str(first) assert (Path(found).parent / second.name).exists() + + +def test_detect_mtp_file_pairs_bpw_qualified_subdir_drafter(tmp_path): + """_extract_quant_label supports bpw-qualified names, so pairing must too.""" + weight = tmp_path / "model-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + drafter = sub / "mtp-model-IQ4_XS-3.53bpw.gguf" + drafter.write_bytes(b"x") + + assert detect_mtp_file(str(weight)) == str(drafter.resolve()) + + +def test_detect_mtp_file_skips_incomplete_split_drafter(tmp_path): + """An incomplete shard set fails llama-server's draft startup, so a + complete copy must win rather than MTP being disabled.""" + weight = tmp_path / "model-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + # Declares two shards but ships only the first. + (sub / "mtp-model-Q4_0-00001-of-00002.gguf").write_bytes(b"x" * 50) + complete = sub / "mtp-model-BF16.gguf" + complete.write_bytes(b"x" * 100) + + assert detect_mtp_file(str(weight)) == str(complete.resolve()) + + +def test_detect_mtp_file_ranks_split_drafter_by_total_size(tmp_path): + """Candidates collapse to shard 1, so a split copy must be summed or it + outranks a smaller single file.""" + weight = tmp_path / "model-Q4_0.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + (sub / "mtp-model-Q8_0-00001-of-00002.gguf").write_bytes(b"x" * 90) + (sub / "mtp-model-Q8_0-00002-of-00002.gguf").write_bytes(b"x" * 90) + smaller = sub / "mtp-model-BF16.gguf" + smaller.write_bytes(b"x" * 100) + + assert detect_mtp_file(str(weight)) == str(smaller.resolve()) diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index ab1ba4f7fe..3b0df808dd 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -17,6 +17,8 @@ if _BACKEND_DIR not in sys.path: from routes.inference import _validate_native_gguf_companion from routes.inference import _request_matches_loaded_settings +from routes.inference import _validate_native_mtp_drafter +from routes.inference import _loaded_is_local_model from core.inference.llama_cpp import LlamaCppBackend from models.inference import LoadRequest @@ -286,3 +288,49 @@ def test_reload_dedup_native_load_with_no_admissible_drafter(tmp_path, monkeypat assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) # An ordinary load would launch the root drafter, so it must reload. assert not _request_matches_loaded_settings(request, backend, None, native_grant_backed = False) + + +def test_native_mtp_drafter_rejects_symlinked_later_shard(tmp_path): + """llama-server opens sibling shards implicitly, so validating only the + launch path would let a later shard escape the permitted directory.""" + weight = tmp_path / "model-Q4_0.gguf" + weight.write_bytes(b"model") + sub = tmp_path / "MTP" + sub.mkdir() + first = sub / "mtp-model-Q4_0-00001-of-00002.gguf" + first.write_bytes(b"draft") + outside = tmp_path / "outside.bin" + outside.write_bytes(b"secret") + try: + (sub / "mtp-model-Q4_0-00002-of-00002.gguf").symlink_to(outside) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + with pytest.raises(HTTPException, match = "regular file"): + _validate_native_mtp_drafter(str(first), str(weight), mtp_search_root = str(tmp_path)) + + +def test_native_mtp_drafter_accepts_regular_shard_set(tmp_path): + weight = tmp_path / "model-Q4_0.gguf" + weight.write_bytes(b"model") + sub = tmp_path / "MTP" + sub.mkdir() + first = sub / "mtp-model-Q4_0-00001-of-00002.gguf" + first.write_bytes(b"draft") + (sub / "mtp-model-Q4_0-00002-of-00002.gguf").write_bytes(b"draft") + + _validate_native_mtp_drafter(str(first), str(weight), mtp_search_root = str(tmp_path)) + + +def test_status_provenance_survives_deleted_model_directory(tmp_path, monkeypatch): + """Provenance is a load-time fact: a directory removed underneath a running + server must not turn a local model into a remote one.""" + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._is_local_model = True + # "outputs/gemma" no longer exists, so is_local_path would call it a repo id. + assert _loaded_is_local_model(backend, False, "outputs/gemma") + + stale = LlamaCppBackend() + assert not _loaded_is_local_model(stale, False, "unsloth/gemma-4-12b") + assert _loaded_is_local_model(stale, True, None) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 6e49294e3f..523b2a7b30 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1301,6 +1301,47 @@ def _colocated_first_split_shard(path: Path) -> tuple[Optional[Path], bool]: return first, first is not None and len(indices) == total +def colocated_split_shards(path: Path) -> tuple[list[Path], bool]: + """Every shard beside *path*, and whether the declared set is complete. + + A non-split path is itself a complete one-file set. Callers that hand a + path to llama-server need this: it opens the sibling shards implicitly, so + an incomplete set fails at startup and every shard needs validating. + """ + match = _GGUF_SPLIT_FILE_RE.match(path.name) + if match is None: + return [path], True + + prefix = match.group("prefix").casefold() + total_text = match.group("total") + total = int(total_text) + if total < 1: + return [], False + + found: dict[int, Path] = {} + try: + for sibling in path.parent.iterdir(): + sibling_match = _GGUF_SPLIT_FILE_RE.match(sibling.name) + if ( + sibling_match is None + or sibling_match.group("prefix").casefold() != prefix + or sibling_match.group("total") != total_text + ): + continue + try: + if not sibling.is_file(): + continue + except OSError: + continue + index = int(sibling_match.group("index")) + if 1 <= index <= total: + found[index] = sibling + except OSError: + return [], False + + return [found[i] for i in sorted(found)], len(found) == total + + def _local_gguf_load_path(path: Path) -> Path: """Choose a loadable local path while preserving complete symlink sets.""" if _GGUF_SPLIT_FILE_RE.match(path.name) is None: @@ -1476,7 +1517,13 @@ def detect_mtp_file( if stem.endswith("-mtp"): stem = stem[: -len("-mtp")] # Full quant vocabulary, not a subset: K/IQ/UD/MXFP drafters pair too. - return re.sub(rf"-(?:{_GGUF_KNOWN_QUANT_RE.pattern})$", "", stem, flags = re.IGNORECASE) + # The optional bpw modifier goes with it, as _extract_quant_label does. + return re.sub( + rf"-(?:{_GGUF_KNOWN_QUANT_RE.pattern})(?:-[0-9]+(?:\.[0-9]+)?bpw)?$", + "", + stem, + flags = re.IGNORECASE, + ) def _drafter_launch_path(candidate: Path) -> str: # llama-server takes shard 1 as the model path, and a split copy must @@ -1497,13 +1544,25 @@ def detect_mtp_file( and (len(weight_name) == len(stem) or not weight_name[len(stem)].isalnum()) ) + def _launchable(candidate: Path) -> bool: + # An incomplete split set makes llama-server fail its draft startup and + # would disable MTP entirely, so skip it and let a complete copy win. + try: + _, complete = colocated_split_shards(candidate) + except OSError: + return False + return complete + def _smallest_first(candidate: Path) -> tuple[int, int, str]: # Cheapest compatible copy wins. Size first: a fixed precision list # ranked unknown quants behind BF16, so a small K-quant lost to a far # larger BF16. Precision breaks size ties, name keeps it stable. + # Candidates are collapsed to shard 1, so a split copy must be summed + # across its shards or it would outrank a smaller single file. name = candidate.name.lower() try: - size = candidate.stat().st_size + shards, _ = colocated_split_shards(candidate) + size = sum(shard.stat().st_size for shard in shards) except OSError: size = sys.maxsize if "-q4_0" in name: @@ -1535,7 +1594,7 @@ def detect_mtp_file( if not _matches_weight(f): continue try: - if f.is_file(): + if f.is_file() and _launchable(f): return _drafter_launch_path(f) except OSError: continue @@ -1577,7 +1636,7 @@ def detect_mtp_file( if not _matches_weight(f): continue try: - if f.is_file(): + if f.is_file() and _launchable(f): # Collapse a split copy to shard 1 before ranking. subdir_candidates.append(_local_gguf_load_path(f)) except OSError: diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index c547f4927b..dedda3f297 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -376,7 +376,11 @@ def test_local_mtp_warning_uses_backend_source_metadata(): route = _read_backend("routes/inference.py") assert route.count("is_local_model = config.is_local") >= 2 - assert "is_local_model = _native_grant_backed" in route + # GGUF status reports the provenance the load recorded. Re-deriving it from + # the filesystem would flip a local model to remote once its directory goes + # away underneath a running server. + assert "llama_backend._is_local_model = bool(native_grant_backed or config.is_local)" in route + assert "is_local_model = _loaded_is_local_model(" in route assert "backend.active_model_name and is_local_path(backend.active_model_name)" in route From 1f7adc6c5ff21f4100827c14f379619bcbd6d23a Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:09:29 -0700 Subject: [PATCH 12/14] Studio: report persisted provenance from the dedup reply too Both review points on the previous revision were correct. - the already_loaded GGUF response still derived provenance from the filesystem, so a deduplicated /load flipped a local model to remote once its directory went away, the same flip already fixed for the status poll. It now reads the persisted value - drop the .gguf suffix from the local classification in the settings sheet. A one-slash org/name.gguf is a repository id, not a file, as _is_direct_gguf_file_ref documents, so that suffix overrode the backend saying remote and offered filesystem placement guidance for a model that downloads The contract now pins both GGUF responses using the provenance helper, and pins the suffix out of the classification. --- studio/backend/routes/inference.py | 5 +++-- .../src/features/chat/chat-settings-sheet.tsx | 12 +++++------- tests/studio/test_model_picker_contracts.py | 11 ++++++++--- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 6f204b7e38..0615485304 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -4509,8 +4509,9 @@ async def _load_model_impl( is_vision = llama_backend._is_vision, is_lora = False, is_gguf = True, - is_local_model = native_grant_backed - or is_local_path(llama_backend.model_identifier), + is_local_model = _loaded_is_local_model( + llama_backend, native_grant_backed, llama_backend.model_identifier + ), is_diffusion = llama_backend.is_diffusion, is_audio = _gguf_is_audio, audio_type = _gguf_audio, diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx index d0a525ca18..ce6fe6d0f5 100644 --- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx +++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx @@ -394,14 +394,12 @@ export function ChatSettingsPanel({ ggufContextLength != null || (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false); // activeModelIsLocal is the backend's own classification and covers native - // picks. activeNativePathToken must not be used here: status reconciliation - // keeps it across a switch to a remote GGUF (no replacement token exists), - // so a stale token would label that remote model local. + // picks. Two things must not decide this: activeNativePathToken, which + // status reconciliation keeps across a switch to a remote GGUF (no + // replacement token exists), and a bare .gguf suffix, since the backend + // reads a one-slash org/name.gguf as a repository id, not a file. const isLocalGguf = - isGguf && - (activeModelIsLocal || - isLocalModelPath(currentCheckpoint ?? "") || - (currentCheckpoint?.toLowerCase().endsWith(".gguf") ?? false)); + isGguf && (activeModelIsLocal || isLocalModelPath(currentCheckpoint ?? "")); const ggufMaxContextLength = useChatRuntimeStore( (s) => s.ggufMaxContextLength, ); diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index dedda3f297..35125f02dd 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -351,9 +351,12 @@ def test_local_mtp_warning_covers_path_and_native_gguf_sources(): assert "isGguf &&" in local.group(0) assert "activeModelIsLocal" in local.group(0) assert "isLocalModelPath" in local.group(0) - # A native token outlives a switch to a remote GGUF, so it must not - # classify the model here; activeModelIsLocal already covers native picks. + # Two signals must not classify the model here, because both mislabel a + # remote GGUF as local: a native token, which outlives a switch to a remote + # model, and a bare .gguf suffix, since a one-slash org/name.gguf is a + # repository id. activeModelIsLocal is the backend's own answer for both. assert "activeNativePathToken" not in local.group(0) + assert ".gguf" not in local.group(0) assert "isLocalGguf" in src.split('specFallbackReason === "drafter_not_found"', 1)[1] @@ -380,7 +383,9 @@ def test_local_mtp_warning_uses_backend_source_metadata(): # the filesystem would flip a local model to remote once its directory goes # away underneath a running server. assert "llama_backend._is_local_model = bool(native_grant_backed or config.is_local)" in route - assert "is_local_model = _loaded_is_local_model(" in route + # Both GGUF responses report it: the status poll and the already_loaded + # dedup reply. Either one re-deriving it reintroduces the flip. + assert route.count("is_local_model = _loaded_is_local_model(") >= 2 assert "backend.active_model_name and is_local_path(backend.active_model_name)" in route From 05932db5ec743673bc754873d6a596b421a8af56 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:29:37 -0700 Subject: [PATCH 13/14] Studio: share one quant vocabulary with the companion search root The review point was correct. _local_gguf_companion_search_root carried its own copy of the quant pattern, which never gained the bpw modifier, so a directory such as IQ4_XS-3.53bpw was not recognised as a quant dir. The search root stayed inside it and the repository-root MTP/ copy was out of scope for discovery, the native fallback and reload dedup alike. It now builds on _GGUF_KNOWN_QUANT_RE plus the optional bpw suffix, so there is a single vocabulary rather than a duplicate that can fall behind again. Promotion is unchanged for every previously matching name and still rejects DeepSeek-V3-UD-Q2_K_XL, Q4_0-extra and Q4_0bpw. --- .../tests/test_mtp_drafter_companion.py | 28 +++++++++++++++++++ studio/backend/utils/models/model_config.py | 14 ++-------- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index bf1ef1a061..95179b86bd 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -799,3 +799,31 @@ def test_detect_mtp_file_ranks_split_drafter_by_total_size(tmp_path): smaller.write_bytes(b"x" * 100) assert detect_mtp_file(str(weight)) == str(smaller.resolve()) + + +def test_companion_search_root_promotes_bpw_quant_directory(tmp_path): + """A bpw-qualified quant directory must resolve to the repository root, or + the repo-root MTP/ copy is never in scope for it.""" + quant_dir = tmp_path / "IQ4_XS-3.53bpw" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"x") + sub = tmp_path / "MTP" + sub.mkdir() + drafter = sub / "mtp-model.gguf" + drafter.write_bytes(b"x") + + # Directory selection and the file inside it agree on the root. + assert _local_gguf_companion_search_root(str(quant_dir), str(weight)) == str(tmp_path) + assert _local_gguf_companion_search_root(str(weight), str(weight)) == str(tmp_path) + assert detect_mtp_file(str(weight), str(tmp_path)) == str(drafter.resolve()) + + +def test_companion_search_root_keeps_non_quant_directories(tmp_path): + """Sharing the quant vocabulary must not widen what gets promoted.""" + for name in ("DeepSeek-V3-UD-Q2_K_XL", "outputs", "Q4_0-extra", "Q4_0bpw"): + directory = tmp_path / name + directory.mkdir() + weight = directory / "model.gguf" + weight.write_bytes(b"x") + assert _local_gguf_companion_search_root(str(directory), str(weight)) == str(directory) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 523b2a7b30..e6c1d0b4d7 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1865,17 +1865,9 @@ def _local_gguf_companion_search_root(selected_path: str, gguf_file: str) -> str selected = Path(selected_path) gguf_path = Path(gguf_file) - quant_dir_re = ( - r"(UD-)?(" - r"MXFP[0-9]+(?:_[A-Z0-9]+)*" - r"|IQ[0-9]+_[A-Z]+(?:_[A-Z0-9]+)?" - r"|TQ[0-9]+_[0-9]+" - r"|Q[0-9]+_K_[A-Z]+" - r"|Q[0-9]+_[0-9]+" - r"|Q[0-9]+_K" - r"|BF16|F16|F32" - r")" - ) + # One quant vocabulary, shared: a local copy of it silently fell behind on + # the bpw modifier, which left IQ4_XS-3.53bpw unrecognised as a quant dir. + quant_dir_re = rf"{_GGUF_KNOWN_QUANT_RE.pattern}(-[0-9]+(?:\.[0-9]+)?bpw)?" search_dir = gguf_path.parent if selected.suffix.lower() == ".gguf" else selected if not search_dir.name: return str(search_dir) From 0b104d8f1f394733787c0362e27d62f59d76e07f Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Mon, 27 Jul 2026 00:19:12 -0700 Subject: [PATCH 14/14] Studio: scan past rejected drafters and reset provenance on model switch Both review points were correct. - detect_mtp_file takes an optional accept callback, so a caller with extra rules keeps scanning its candidates in preference order. A native load whose size-preferred MTP/ copy resolved out of the grant was treating that rejection as exhaustion and disabling MTP, even with a valid copy beside it. Both the load path and reload dedup now pass their admissibility check straight in, which also removes the manual two-step retry - clear activeModelIsLocal and specFallbackReason in setCheckpoint when the checkpoint really changes. Both describe the model being replaced, so a selection change was classifying the newly chosen model by the previous one's provenance. They are cleared together: dropping one alone pairs a stale reason with the wrong recovery text, which is the flip fixed earlier for the no-active-model branch. The load or status response reseeds both. --- studio/backend/routes/inference.py | 26 +++++----- .../tests/test_native_gguf_companion.py | 50 +++++++++++++++++++ studio/backend/utils/models/model_config.py | 16 ++++-- .../chat/stores/chat-runtime-store.ts | 12 ++++- tests/studio/test_model_picker_contracts.py | 8 +++ 5 files changed, 95 insertions(+), 17 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0615485304..f8cc2d9a1b 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3423,18 +3423,18 @@ def _request_matches_loaded_settings( # copy and, failing that, to no drafter at all. An ordinary # load reaches the root drafter, so it keeps root-first # detection and reloads when one appears. - if detected and not _native_gguf_companion_usable( - detected, llama_backend.gguf_path, mtp_search_root = companion_root - ): + def _usable(candidate: str) -> bool: + return _native_gguf_companion_usable( + candidate, llama_backend.gguf_path, mtp_search_root = companion_root + ) + + if detected and not _usable(detected): detected = detect_mtp_file( llama_backend.gguf_path, search_root = companion_root, skip_root = True, + accept = _usable, ) - if detected and not _native_gguf_companion_usable( - detected, llama_backend.gguf_path, mtp_search_root = companion_root - ): - detected = None stored = llama_backend.mtp_draft_path try: detected_resolved = Path(detected).resolve() if detected else None @@ -4777,21 +4777,21 @@ async def _load_model_impl( if not _mtp_allowed(config.gguf_mtp_file): # The preferred root drafter is out of bounds for a - # grant on a quant subdir, but its MTP/ copy may - # not be. Use that before dropping MTP entirely. + # grant on a quant subdir, but an MTP/ copy may not + # be. Scan them in preference order rather than + # dropping MTP on the first rejection. fallback = detect_mtp_file( config.gguf_file, search_root = mtp_search_root, skip_root = True, + accept = _mtp_allowed, ) - if fallback and _mtp_allowed(fallback): + if fallback: logger.info( "Using MTP subdirectory drafter for native load: %s", fallback, ) - config.gguf_mtp_file = fallback - else: - config.gguf_mtp_file = None + config.gguf_mtp_file = fallback _source_load_kwargs = dict( gguf_path = config.gguf_file, mmproj_path = config.gguf_mmproj_file, diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index 3b0df808dd..3a7b4b0caf 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -19,6 +19,8 @@ from routes.inference import _validate_native_gguf_companion from routes.inference import _request_matches_loaded_settings from routes.inference import _validate_native_mtp_drafter from routes.inference import _loaded_is_local_model +from routes.inference import _native_gguf_companion_usable +from utils.models.model_config import detect_mtp_file from core.inference.llama_cpp import LlamaCppBackend from models.inference import LoadRequest @@ -334,3 +336,51 @@ def test_status_provenance_survives_deleted_model_directory(tmp_path, monkeypatc stale = LlamaCppBackend() assert not _loaded_is_local_model(stale, False, "unsloth/gemma-4-12b") assert _loaded_is_local_model(stale, True, None) + + +def test_native_load_skips_rejected_mtp_candidate_for_next_one(tmp_path): + """MTP/ can hold several compatible copies. If the size-preferred one is + out of the grant, the next must be tried instead of disabling MTP.""" + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + outside = tmp_path.parent / "outside-blob.gguf" + outside.write_bytes(b"d") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + try: + (companion_dir / "mtp-model-Q4_0.gguf").symlink_to(outside) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + larger = companion_dir / "mtp-model-Q8_0.gguf" + larger.write_bytes(b"d" * 5000) + + def _usable(candidate: str) -> bool: + return _native_gguf_companion_usable(candidate, str(weight), mtp_search_root = str(tmp_path)) + + # Preferred by size, but it resolves out of the permitted directory. + assert not _usable(detect_mtp_file(str(weight), str(tmp_path), skip_root = True)) + assert detect_mtp_file(str(weight), str(tmp_path), skip_root = True, accept = _usable) == str( + larger.resolve() + ) + + +def test_native_load_returns_none_when_no_candidate_passes(tmp_path): + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + outside = tmp_path.parent / "outside-only.gguf" + outside.write_bytes(b"d") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + try: + (companion_dir / "mtp-model-Q4_0.gguf").symlink_to(outside) + except OSError as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + def _usable(candidate: str) -> bool: + return _native_gguf_companion_usable(candidate, str(weight), mtp_search_root = str(tmp_path)) + + assert detect_mtp_file(str(weight), str(tmp_path), skip_root = True, accept = _usable) is None diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index e6c1d0b4d7..d379e7ad0a 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -29,7 +29,7 @@ import re import subprocess import sys from pathlib import Path -from typing import List, Tuple +from typing import Callable, List, Tuple import hashlib import json import threading @@ -1486,6 +1486,7 @@ def detect_mtp_file( path: str, search_root: Optional[str] = None, skip_root: bool = False, + accept: Optional[Callable[[str], bool]] = None, ) -> Optional[str]: """Find the separate MTP drafter (``mtp-*.gguf``) for a local GGUF model. @@ -1505,6 +1506,9 @@ def detect_mtp_file( ``skip_root`` scans only ``MTP/``, for callers that must discard an out-of-bounds root drafter and still want the subdir copy (native loads). + ``accept`` filters candidates in preference order, so a caller with extra + rules (a native lease) keeps scanning instead of treating the first + rejection as no drafter at all. """ def _pairing_stem(name: str) -> str: @@ -1594,10 +1598,14 @@ def detect_mtp_file( if not _matches_weight(f): continue try: - if f.is_file() and _launchable(f): - return _drafter_launch_path(f) + if not (f.is_file() and _launchable(f)): + continue + launch = _drafter_launch_path(f) except OSError: continue + if accept is not None and not accept(launch): + continue + return launch subdir_candidates: list[Path] = [] for d in dirs: @@ -1647,6 +1655,8 @@ def detect_mtp_file( resolved = _drafter_launch_path(candidate) except OSError: continue + if accept is not None and not accept(resolved): + continue logger.info(f"Detected MTP subdirectory drafter: {resolved}") return resolved return None diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 9409945072..7a7475406f 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -1538,7 +1538,17 @@ export const useChatRuntimeStore = create((set, get) => ({ maxTokens: nextMaxTokens, }, activeGgufVariant: ggufVariant ?? null, - ...(checkpointChanged ? { contextUsage: null } : {}), + // Provenance and the spec-fallback reason both describe the model + // being replaced, so they go together on a real change. Dropping only + // one leaves the settings sheet pairing a stale reason with the wrong + // recovery text. The load or status response reseeds both. + ...(checkpointChanged + ? { + contextUsage: null, + activeModelIsLocal: false, + specFallbackReason: null, + } + : {}), }; }), setActiveThreadId: (activeThreadId) => diff --git a/tests/studio/test_model_picker_contracts.py b/tests/studio/test_model_picker_contracts.py index 35125f02dd..3450bfdc74 100644 --- a/tests/studio/test_model_picker_contracts.py +++ b/tests/studio/test_model_picker_contracts.py @@ -357,6 +357,14 @@ def test_local_mtp_warning_covers_path_and_native_gguf_sources(): # repository id. activeModelIsLocal is the backend's own answer for both. assert "activeNativePathToken" not in local.group(0) assert ".gguf" not in local.group(0) + + # Switching models must drop both together: a kept flag would classify the + # newly selected model by the old one's provenance. + store = _read("features/chat/stores/chat-runtime-store.ts") + reset = re.search(r"setCheckpoint: \(modelId, ggufVariant\) =>.*?\}\),", store, re.S) + assert reset + assert "activeModelIsLocal: false" in reset.group(0) + assert "specFallbackReason: null" in reset.group(0) assert "isLocalGguf" in src.split('specFallbackReason === "drafter_not_found"', 1)[1]