Studio: pin llama.cpp update apply to the release the banner offered (#7112)

* Studio: pin llama.cpp update apply to the release the banner offered

* Document the pinned walk-back trade-off and cover win32

* Trim the pin comments

* Studio: verify a pinned llama.cpp update landed on the pinned release

The pin passes --published-release-tag so the installer resolves exactly the offered release. Also verify the result: if the post-install marker stays on the pinned repo but reports a different tag, the installer ignored the pin, so fail with a retryable error instead of a false success. A Vulkan/Intel host legitimately reroutes fork to upstream and drops the pin, so the check is scoped to the pinned repo.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: danielhanchen <michaelhan2050@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

studio/backend/tests/test_llama_cpp_update.py

@@ -393,6 +393,9 @@ def test_start_update_source_build_installs_prebuilt(monkeypatch, tmp_path):
     assert "--llama-tag" in cmd and "latest" in cmd
     assert cmd[cmd.index("--rocm-gfx") + 1] == "gfx110x"
     assert "--simple-policy" not in cmd and "--cpu-fallback" not in cmd
+    # No pin: source-build detection and the unpinned apply share the same
+    # "latest" resolver, so they already agree.
+    assert "--published-release-tag" not in cmd
 
 
 def test_start_update_happy_path(monkeypatch, tmp_path):
@@ -519,6 +522,57 @@ def test_start_update_reports_full_release_tag(monkeypatch, tmp_path):
     assert "Updated llama.cpp to b9596-mix-e6f2453." in job["message"]
 
 
+def _run_start_update_to_completion():
+    res = upd.start_update()
+    assert res["started"] is True
+    deadline = time.time() + 10
+    while time.time() < deadline:
+        job = upd.get_update_status()["job"]
+        if job["state"] in ("success", "error"):
+            return job
+        time.sleep(0.05)
+    return upd.get_update_status()["job"]
+
+
+def test_start_update_pinned_tag_mismatch_fails(monkeypatch, tmp_path):
+    # Installer stays on the pinned repo but produces a different tag -> it
+    # ignored the pin (the silent mismatch this pin exists to prevent). Fail loud.
+    monkeypatch.setattr(sys, "platform", "linux")
+    install_dir = tmp_path / "llama.cpp"
+    binary = _write_install(install_dir, "b9595")
+    monkeypatch.setattr(upd, "_find_binary", lambda: binary)
+    monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")
+    monkeypatch.setattr(
+        freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9601-mix-a0e2906"
+    )
+    _patch_installer_popen(
+        monkeypatch,
+        on_start = lambda cmd: _write_install(install_dir, "b9500", release_tag = "b9500-mix-deadbee"),
+    )
+    job = _run_start_update_to_completion()
+    assert job["state"] == "error", job
+    assert "b9601-mix-a0e2906" in (job["error"] or "")
+
+
+def test_start_update_pinned_reroute_to_other_repo_ok(monkeypatch, tmp_path):
+    # A Vulkan/Intel host reroutes fork->upstream and drops the pin, installing a
+    # different-repo tag. Legitimate: the pin check must not flag the repo switch.
+    monkeypatch.setattr(sys, "platform", "linux")
+    install_dir = tmp_path / "llama.cpp"
+    binary = _write_install(install_dir, "b9595", repo = "unslothai/llama.cpp")
+    monkeypatch.setattr(upd, "_find_binary", lambda: binary)
+    monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")
+    monkeypatch.setattr(
+        freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9601-mix-a0e2906"
+    )
+    _patch_installer_popen(
+        monkeypatch,
+        on_start = lambda cmd: _write_install(install_dir, "b9601", repo = "ggml-org/llama.cpp"),
+    )
+    job = _run_start_update_to_completion()
+    assert job["state"] == "success", job
+
+
 def test_start_update_installer_failure_reports_error(monkeypatch, tmp_path):
     install_dir = tmp_path / "llama.cpp"
     binary = _write_install(install_dir, "b9493")
@@ -663,6 +717,33 @@ def test_install_cmd_cuda_marker_minimal_and_backward_compatible(monkeypatch, tm
     assert "--cpu-fallback" not in cmd
 
 
+def test_install_cmd_pins_offered_release_tag(monkeypatch, tmp_path):
+    # Apply must install exactly the release the banner offered. The installer's
+    # own "latest" comes from commit-date-ordered sources, which can lag the
+    # published_at-newest tag detection picked; unpinned, that lag makes Update
+    # reinstall the current build while the banner never clears.
+    monkeypatch.setattr(sys, "platform", "linux")
+    cmd = _capture_install_cmd(monkeypatch, tmp_path, latest = "b9601-mix-a0e2906")
+    # The full release identity is pinned, not the bare upstream base.
+    assert cmd[cmd.index("--published-release-tag") + 1] == "b9601-mix-a0e2906"
+
+
+def test_install_cmd_pins_on_windows(monkeypatch, tmp_path):
+    # The darwin exemption must not leak to other platforms.
+    monkeypatch.setattr(sys, "platform", "win32")
+    cmd = _capture_install_cmd(monkeypatch, tmp_path)
+    assert cmd[cmd.index("--published-release-tag") + 1] == "b9518"
+
+
+def test_install_cmd_does_not_pin_on_macos(monkeypatch, tmp_path):
+    # A pinned tag disables the installer's older-release walk-back, which macOS
+    # needs to skip prebuilts built for a newer macOS than the host.
+    monkeypatch.setattr(sys, "platform", "darwin")
+    cmd = _capture_install_cmd(monkeypatch, tmp_path)
+    assert "--published-release-tag" not in cmd
+    assert "--llama-tag" in cmd and "latest" in cmd
+
+
 # --- refusal + maintenance-state coordination ---
 
 

studio/backend/utils/llama_cpp_update.py

@@ -473,9 +473,18 @@ def _rocm_install_args(asset: Optional[str]) -> list[str]:
     return ["--has-rocm"]
 
 
-def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path) -> None:
+def _run_update(
+    install_dir: Path,
+    repo: str,
+    asset: Optional[str],
+    script: Path,
+    pin_release_tag: Optional[str] = None,
+) -> None:
     """Worker: put the backend into a maintenance state, run the installer for
-    the latest prebuilt, then refresh caches so the next load uses the new build."""
+    the latest prebuilt, then refresh caches so the next load uses the new build.
+
+    pin_release_tag pins the installer to that exact published release instead
+    of letting it re-resolve "latest" itself (see start_update for why)."""
     backend = None
     model_was_active = False
     try:
@@ -510,6 +519,8 @@ def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path
             "--published-repo",
             repo,
         ]
+        if pin_release_tag:
+            cmd.extend(["--published-release-tag", pin_release_tag])
         cmd.extend(_rocm_install_args(asset))
         logger.info("llama update: installing", cmd = " ".join(cmd))
         # Stream progress lines into job["progress"].
@@ -569,6 +580,16 @@ def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path
         new_marker = read_install_marker(_find_binary())
         new_tag = (new_marker or {}).get("release_tag") or (new_marker or {}).get("tag")
 
+        # Pinned install must land on that exact release; a same-repo mismatch
+        # means the pin was ignored (Vulkan/Intel reroute to another repo is fine).
+        if (
+            pin_release_tag
+            and new_tag
+            and (new_marker or {}).get("published_repo") == repo
+            and new_tag != pin_release_tag
+        ):
+            raise RuntimeError(f"pinned release {pin_release_tag} but installer produced {new_tag}")
+
         with _job_lock:
             _job.update(
                 state = _JOB_SUCCESS,
@@ -650,6 +671,13 @@ def start_update() -> dict:
         repo = marker.get("published_repo") or DEFAULT_PUBLISHED_REPO
         from_tag = marker.get("tag") or marker.get("release_tag")
         asset = marker.get("asset")
+        # Install exactly the release the banner offered: the installer's own
+        # "latest" is commit-date ordered and can lag the published_at pick
+        # above, reinstalling the current build in a loop (the #6219 class).
+        # Not on macOS, which needs the older-release walk-back a pin disables
+        # (skipping too-new prebuilts); elsewhere an unusable latest now fails
+        # the job loudly (retryable) instead of walking back.
+        pin_release_tag = None if sys.platform == "darwin" else status.get("latest_tag")
     else:
         # Source build / custom path: only proceed when the same detection logic
         # would offer the update (prebuilt exists, install is behind, root is
@@ -677,6 +705,9 @@ def start_update() -> dict:
         repo = (res or {}).get("repo") or DEFAULT_PUBLISHED_REPO
         from_tag = None
         asset = (res or {}).get("asset")
+        # No pin: source-build detection resolves via --resolve-prebuilt latest,
+        # the same resolver the unpinned apply uses, so the two already agree.
+        pin_release_tag = None
 
     if install_dir is None:
         return {
@@ -704,7 +735,7 @@ def start_update() -> dict:
 
     thread = threading.Thread(
         target = _run_update,
-        args = (install_dir, repo, asset, script),
+        args = (install_dir, repo, asset, script, pin_release_tag),
         name = "llama-cpp-update",
         daemon = True,
     )