Studio: keep sharded MTP snapshot paths and gate dedup on native loads
Both review points on the previous revision were correct. - stop resolving a split MTP/ drafter to its blob target. Snapshot symlinks are how HF stores shards, and the blob has no sibling shard names, so --model-draft could not load. _local_gguf_load_path already preserves the snapshot path; the later resolve() was undoing it. Single file drafters still resolve as before - restrict the reload deduplication fallback to native loads. An ordinary local load can reach a root drafter added after the fact and must reload to pick it up; only a native load, whose root candidate is outside the lease, keeps running the MTP/ copy Tests cover the snapshot shard path and both deduplication routes.
This commit is contained in:
parent
bb8ab88f6e
commit
fc90affb29
4 changed files with 70 additions and 3 deletions
|
|
@ -3215,6 +3215,7 @@ def _request_matches_loaded_settings(
|
|||
request: LoadRequest,
|
||||
llama_backend: LlamaCppBackend,
|
||||
effective_chat_template_override: Optional[str] = None,
|
||||
native_grant_backed: bool = False,
|
||||
) -> bool:
|
||||
"""True iff every runtime setting on the request matches the loaded server.
|
||||
Caller has already checked model+variant+is_loaded. See #5401.
|
||||
|
|
@ -3368,7 +3369,11 @@ def _request_matches_loaded_settings(
|
|||
# A native load whose root drafter was out of bounds runs the
|
||||
# MTP/ fallback instead, so root-first detection never equals
|
||||
# what launched. Accept the subdir copy as current too, else
|
||||
# that layout reloads on every apply.
|
||||
# that layout reloads on every apply. Native only: an ordinary
|
||||
# load can reach the root drafter, so a newly added one must
|
||||
# still reload.
|
||||
if not native_grant_backed:
|
||||
return False
|
||||
fallback = detect_mtp_file(
|
||||
llama_backend.gguf_path, search_root = companion_root, skip_root = True
|
||||
)
|
||||
|
|
@ -4419,6 +4424,7 @@ async def _load_model_impl(
|
|||
request,
|
||||
llama_backend,
|
||||
effective_chat_template_override,
|
||||
native_grant_backed = native_grant_backed,
|
||||
)
|
||||
# Skip if a prior audio probe failed -- let load_model retry.
|
||||
and getattr(llama_backend, "_audio_probed", True)
|
||||
|
|
|
|||
|
|
@ -689,3 +689,31 @@ def test_detect_mtp_file_pairs_k_quant_subdir_drafter(tmp_path):
|
|||
drafter.write_bytes(b"x")
|
||||
|
||||
assert detect_mtp_file(str(weight)) == str(drafter.resolve())
|
||||
|
||||
|
||||
def test_detect_mtp_file_keeps_snapshot_path_for_sharded_subdir_drafter(tmp_path):
|
||||
"""A split copy stored as HF snapshot symlinks must launch from the
|
||||
snapshot path: the blob target has no sibling shard names."""
|
||||
blobs = tmp_path / "blobs"
|
||||
snapshot = tmp_path / "snapshots" / "abc"
|
||||
sub = snapshot / "MTP"
|
||||
blobs.mkdir(parents = True)
|
||||
sub.mkdir(parents = True)
|
||||
|
||||
(blobs / "sha_weight").write_bytes(b"w")
|
||||
weight = snapshot / "model-Q4_0.gguf"
|
||||
try:
|
||||
weight.symlink_to(blobs / "sha_weight")
|
||||
except OSError:
|
||||
pytest.skip("symlinks unavailable")
|
||||
|
||||
first = sub / "mtp-model-Q4_0-00001-of-00002.gguf"
|
||||
second = sub / "mtp-model-Q4_0-00002-of-00002.gguf"
|
||||
(blobs / "sha_1").write_bytes(b"d" * 4096)
|
||||
(blobs / "sha_2").write_bytes(b"d")
|
||||
first.symlink_to(blobs / "sha_1")
|
||||
second.symlink_to(blobs / "sha_2")
|
||||
|
||||
found = detect_mtp_file(str(weight), str(snapshot))
|
||||
assert found == str(first)
|
||||
assert (Path(found).parent / second.name).exists()
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ def test_reload_dedup_accepts_native_subdir_fallback(tmp_path, monkeypatch):
|
|||
backend._mtp_draft_path = str(companion)
|
||||
|
||||
request = LoadRequest(model_path = str(weight))
|
||||
assert _request_matches_loaded_settings(request, backend)
|
||||
assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True)
|
||||
|
||||
|
||||
def test_reload_dedup_still_reloads_when_drafter_disappears(tmp_path, monkeypatch):
|
||||
|
|
@ -237,3 +237,32 @@ def test_reload_dedup_still_reloads_when_drafter_disappears(tmp_path, monkeypatc
|
|||
companion.unlink()
|
||||
request = LoadRequest(model_path = str(weight))
|
||||
assert not _request_matches_loaded_settings(request, backend)
|
||||
|
||||
|
||||
def test_reload_dedup_reloads_for_ordinary_load_when_root_drafter_appears(tmp_path, monkeypatch):
|
||||
"""The native fallback exception must not swallow a newly added root
|
||||
drafter on an ordinary local load, which can reach it."""
|
||||
quant_dir = tmp_path / "Q4_0"
|
||||
quant_dir.mkdir()
|
||||
weight = quant_dir / "model.gguf"
|
||||
weight.write_bytes(b"model")
|
||||
companion_dir = tmp_path / "MTP"
|
||||
companion_dir.mkdir()
|
||||
companion = companion_dir / "mtp-model-Q4_0.gguf"
|
||||
companion.write_bytes(b"draft")
|
||||
|
||||
monkeypatch.setattr(LlamaCppBackend, "_kill_orphaned_servers", staticmethod(lambda: 0))
|
||||
backend = LlamaCppBackend()
|
||||
backend._gguf_path = str(weight)
|
||||
backend._mtp_draft_path = str(companion)
|
||||
|
||||
request = LoadRequest(model_path = str(weight))
|
||||
# No root drafter yet: both routes dedupe.
|
||||
assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True)
|
||||
assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = False)
|
||||
|
||||
(tmp_path / "mtp-model.gguf").write_bytes(b"root drafter")
|
||||
# Native cannot reach the root drafter, so the subdir copy stays current.
|
||||
assert _request_matches_loaded_settings(request, backend, None, native_grant_backed = True)
|
||||
# An ordinary load would pick the root drafter, so it must reload.
|
||||
assert not _request_matches_loaded_settings(request, backend, None, native_grant_backed = False)
|
||||
|
|
|
|||
|
|
@ -1573,7 +1573,11 @@ def detect_mtp_file(
|
|||
|
||||
for candidate in sorted(dict.fromkeys(subdir_candidates), key = _smallest_first):
|
||||
try:
|
||||
resolved = candidate.resolve()
|
||||
# A split copy keeps its snapshot path: resolving to the blob
|
||||
# drops the sibling shard names llama-server needs to find.
|
||||
resolved = (
|
||||
candidate if _GGUF_SPLIT_FILE_RE.match(candidate.name) else candidate.resolve()
|
||||
)
|
||||
except OSError:
|
||||
continue
|
||||
logger.info(f"Detected MTP subdirectory drafter: {resolved}")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue