From fc90affb29ce3bf1e95e836781bff00688325326 Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:20:24 -0700 Subject: [PATCH] Studio: keep sharded MTP snapshot paths and gate dedup on native loads Both review points on the previous revision were correct. - stop resolving a split MTP/ drafter to its blob target. Snapshot symlinks are how HF stores shards, and the blob has no sibling shard names, so --model-draft could not load. _local_gguf_load_path already preserves the snapshot path; the later resolve() was undoing it. Single file drafters still resolve as before - restrict the reload deduplication fallback to native loads. An ordinary local load can reach a root drafter added after the fact and must reload to pick it up; only a native load, whose root candidate is outside the lease, keeps running the MTP/ copy Tests cover the snapshot shard path and both deduplication routes. --- studio/backend/routes/inference.py | 8 ++++- .../tests/test_mtp_drafter_companion.py | 28 +++++++++++++++++ .../tests/test_native_gguf_companion.py | 31 ++++++++++++++++++- studio/backend/utils/models/model_config.py | 6 +++- 4 files changed, 70 insertions(+), 3 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index f26561673b..2b98d06aff 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -3215,6 +3215,7 @@ def _request_matches_loaded_settings( request: LoadRequest, llama_backend: LlamaCppBackend, effective_chat_template_override: Optional[str] = None, + native_grant_backed: bool = False, ) -> bool: """True iff every runtime setting on the request matches the loaded server. Caller has already checked model+variant+is_loaded. See #5401. @@ -3368,7 +3369,11 @@ def _request_matches_loaded_settings( # A native load whose root drafter was out of bounds runs the # MTP/ fallback instead, so root-first detection never equals # what launched. Accept the subdir copy as current too, else - # that layout reloads on every apply. + # that layout reloads on every apply. Native only: an ordinary + # load can reach the root drafter, so a newly added one must + # still reload. + if not native_grant_backed: + return False fallback = detect_mtp_file( llama_backend.gguf_path, search_root = companion_root, skip_root = True ) @@ -4419,6 +4424,7 @@ async def _load_model_impl( request, llama_backend, effective_chat_template_override, + native_grant_backed = native_grant_backed, ) # Skip if a prior audio probe failed -- let load_model retry. and getattr(llama_backend, "_audio_probed", True) diff --git a/studio/backend/tests/test_mtp_drafter_companion.py b/studio/backend/tests/test_mtp_drafter_companion.py index 2ffeb62d85..15705753b4 100644 --- a/studio/backend/tests/test_mtp_drafter_companion.py +++ b/studio/backend/tests/test_mtp_drafter_companion.py @@ -689,3 +689,31 @@ def test_detect_mtp_file_pairs_k_quant_subdir_drafter(tmp_path): drafter.write_bytes(b"x") assert detect_mtp_file(str(weight)) == str(drafter.resolve()) + + +def test_detect_mtp_file_keeps_snapshot_path_for_sharded_subdir_drafter(tmp_path): + """A split copy stored as HF snapshot symlinks must launch from the + snapshot path: the blob target has no sibling shard names.""" + blobs = tmp_path / "blobs" + snapshot = tmp_path / "snapshots" / "abc" + sub = snapshot / "MTP" + blobs.mkdir(parents = True) + sub.mkdir(parents = True) + + (blobs / "sha_weight").write_bytes(b"w") + weight = snapshot / "model-Q4_0.gguf" + try: + weight.symlink_to(blobs / "sha_weight") + except OSError: + pytest.skip("symlinks unavailable") + + first = sub / "mtp-model-Q4_0-00001-of-00002.gguf" + second = sub / "mtp-model-Q4_0-00002-of-00002.gguf" + (blobs / "sha_1").write_bytes(b"d" * 4096) + (blobs / "sha_2").write_bytes(b"d") + first.symlink_to(blobs / "sha_1") + second.symlink_to(blobs / "sha_2") + + found = detect_mtp_file(str(weight), str(snapshot)) + assert found == str(first) + assert (Path(found).parent / second.name).exists() diff --git a/studio/backend/tests/test_native_gguf_companion.py b/studio/backend/tests/test_native_gguf_companion.py index 105f7b5e1a..24f0553580 100644 --- a/studio/backend/tests/test_native_gguf_companion.py +++ b/studio/backend/tests/test_native_gguf_companion.py @@ -215,7 +215,7 @@ def test_reload_dedup_accepts_native_subdir_fallback(tmp_path, monkeypatch): backend._mtp_draft_path = str(companion) request = LoadRequest(model_path = str(weight)) - assert _request_matches_loaded_settings(request, backend) + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) def test_reload_dedup_still_reloads_when_drafter_disappears(tmp_path, monkeypatch): @@ -237,3 +237,32 @@ def test_reload_dedup_still_reloads_when_drafter_disappears(tmp_path, monkeypatc companion.unlink() request = LoadRequest(model_path = str(weight)) assert not _request_matches_loaded_settings(request, backend) + + +def test_reload_dedup_reloads_for_ordinary_load_when_root_drafter_appears(tmp_path, monkeypatch): + """The native fallback exception must not swallow a newly added root + drafter on an ordinary local load, which can reach it.""" + quant_dir = tmp_path / "Q4_0" + quant_dir.mkdir() + weight = quant_dir / "model.gguf" + weight.write_bytes(b"model") + companion_dir = tmp_path / "MTP" + companion_dir.mkdir() + companion = companion_dir / "mtp-model-Q4_0.gguf" + companion.write_bytes(b"draft") + + monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0)) + backend = LlamaCppBackend() + backend._gguf_path = str(weight) + backend._mtp_draft_path = str(companion) + + request = LoadRequest(model_path = str(weight)) + # No root drafter yet: both routes dedupe. + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = False) + + (tmp_path / "mtp-model.gguf").write_bytes(b"root drafter") + # Native cannot reach the root drafter, so the subdir copy stays current. + assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True) + # An ordinary load would pick the root drafter, so it must reload. + assert not _request_matches_loaded_settings(request, backend, None, native_grant_backed = False) diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py index 1a6036765a..8f1384df27 100644 --- a/studio/backend/utils/models/model_config.py +++ b/studio/backend/utils/models/model_config.py @@ -1573,7 +1573,11 @@ def detect_mtp_file( for candidate in sorted(dict.fromkeys(subdir_candidates), key = _smallest_first): try: - resolved = candidate.resolve() + # A split copy keeps its snapshot path: resolving to the blob + # drops the sibling shard names llama-server needs to find. + resolved = ( + candidate if _GGUF_SPLIT_FILE_RE.match(candidate.name) else candidate.resolve() + ) except OSError: continue logger.info(f"Detected MTP subdirectory drafter: {resolved}")