From 1e0d5ec6a9349a8fa792f013ec7706aace33f45a Mon Sep 17 00:00:00 2001 From: oobabooga Date: Tue, 14 Jul 2026 10:45:16 -0300 Subject: [PATCH] Studio: pin llama.cpp update apply to the release the banner offered (#7112) * Studio: pin llama.cpp update apply to the release the banner offered * Document the pinned walk-back trade-off and cover win32 * Trim the pin comments * Studio: verify a pinned llama.cpp update landed on the pinned release The pin passes --published-release-tag so the installer resolves exactly the offered release. Also verify the result: if the post-install marker stays on the pinned repo but reports a different tag, the installer ignored the pin, so fail with a retryable error instead of a false success. A Vulkan/Intel host legitimately reroutes fork to upstream and drops the pin, so the check is scoped to the pinned repo. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: danielhanchen Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/tests/test_llama_cpp_update.py | 81 +++++++++++++++++++ studio/backend/utils/llama_cpp_update.py | 37 ++++++++- 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index f405ebcbd1..83ea07a066 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -393,6 +393,9 @@ def test_start_update_source_build_installs_prebuilt(monkeypatch, tmp_path): assert "--llama-tag" in cmd and "latest" in cmd assert cmd[cmd.index("--rocm-gfx") + 1] == "gfx110x" assert "--simple-policy" not in cmd and "--cpu-fallback" not in cmd + # No pin: source-build detection and the unpinned apply share the same + # "latest" resolver, so they already agree. + assert "--published-release-tag" not in cmd def test_start_update_happy_path(monkeypatch, tmp_path): @@ -519,6 +522,57 @@ def test_start_update_reports_full_release_tag(monkeypatch, tmp_path): assert "Updated llama.cpp to b9596-mix-e6f2453." in job["message"] +def _run_start_update_to_completion(): + res = upd.start_update() + assert res["started"] is True + deadline = time.time() + 10 + while time.time() < deadline: + job = upd.get_update_status()["job"] + if job["state"] in ("success", "error"): + return job + time.sleep(0.05) + return upd.get_update_status()["job"] + + +def test_start_update_pinned_tag_mismatch_fails(monkeypatch, tmp_path): + # Installer stays on the pinned repo but produces a different tag -> it + # ignored the pin (the silent mismatch this pin exists to prevent). Fail loud. + monkeypatch.setattr(sys, "platform", "linux") + install_dir = tmp_path / "llama.cpp" + binary = _write_install(install_dir, "b9595") + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr( + freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9601-mix-a0e2906" + ) + _patch_installer_popen( + monkeypatch, + on_start = lambda cmd: _write_install(install_dir, "b9500", release_tag = "b9500-mix-deadbee"), + ) + job = _run_start_update_to_completion() + assert job["state"] == "error", job + assert "b9601-mix-a0e2906" in (job["error"] or "") + + +def test_start_update_pinned_reroute_to_other_repo_ok(monkeypatch, tmp_path): + # A Vulkan/Intel host reroutes fork->upstream and drops the pin, installing a + # different-repo tag. Legitimate: the pin check must not flag the repo switch. + monkeypatch.setattr(sys, "platform", "linux") + install_dir = tmp_path / "llama.cpp" + binary = _write_install(install_dir, "b9595", repo = "unslothai/llama.cpp") + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr( + freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9601-mix-a0e2906" + ) + _patch_installer_popen( + monkeypatch, + on_start = lambda cmd: _write_install(install_dir, "b9601", repo = "ggml-org/llama.cpp"), + ) + job = _run_start_update_to_completion() + assert job["state"] == "success", job + + def test_start_update_installer_failure_reports_error(monkeypatch, tmp_path): install_dir = tmp_path / "llama.cpp" binary = _write_install(install_dir, "b9493") @@ -663,6 +717,33 @@ def test_install_cmd_cuda_marker_minimal_and_backward_compatible(monkeypatch, tm assert "--cpu-fallback" not in cmd +def test_install_cmd_pins_offered_release_tag(monkeypatch, tmp_path): + # Apply must install exactly the release the banner offered. The installer's + # own "latest" comes from commit-date-ordered sources, which can lag the + # published_at-newest tag detection picked; unpinned, that lag makes Update + # reinstall the current build while the banner never clears. + monkeypatch.setattr(sys, "platform", "linux") + cmd = _capture_install_cmd(monkeypatch, tmp_path, latest = "b9601-mix-a0e2906") + # The full release identity is pinned, not the bare upstream base. + assert cmd[cmd.index("--published-release-tag") + 1] == "b9601-mix-a0e2906" + + +def test_install_cmd_pins_on_windows(monkeypatch, tmp_path): + # The darwin exemption must not leak to other platforms. + monkeypatch.setattr(sys, "platform", "win32") + cmd = _capture_install_cmd(monkeypatch, tmp_path) + assert cmd[cmd.index("--published-release-tag") + 1] == "b9518" + + +def test_install_cmd_does_not_pin_on_macos(monkeypatch, tmp_path): + # A pinned tag disables the installer's older-release walk-back, which macOS + # needs to skip prebuilts built for a newer macOS than the host. + monkeypatch.setattr(sys, "platform", "darwin") + cmd = _capture_install_cmd(monkeypatch, tmp_path) + assert "--published-release-tag" not in cmd + assert "--llama-tag" in cmd and "latest" in cmd + + # --- refusal + maintenance-state coordination --- diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index 1bcbfbf95a..f6d3635301 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -473,9 +473,18 @@ def _rocm_install_args(asset: Optional[str]) -> list[str]: return ["--has-rocm"] -def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path) -> None: +def _run_update( + install_dir: Path, + repo: str, + asset: Optional[str], + script: Path, + pin_release_tag: Optional[str] = None, +) -> None: """Worker: put the backend into a maintenance state, run the installer for - the latest prebuilt, then refresh caches so the next load uses the new build.""" + the latest prebuilt, then refresh caches so the next load uses the new build. + + pin_release_tag pins the installer to that exact published release instead + of letting it re-resolve "latest" itself (see start_update for why).""" backend = None model_was_active = False try: @@ -510,6 +519,8 @@ def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path "--published-repo", repo, ] + if pin_release_tag: + cmd.extend(["--published-release-tag", pin_release_tag]) cmd.extend(_rocm_install_args(asset)) logger.info("llama update: installing", cmd = " ".join(cmd)) # Stream progress lines into job["progress"]. @@ -569,6 +580,16 @@ def _run_update(install_dir: Path, repo: str, asset: Optional[str], script: Path new_marker = read_install_marker(_find_binary()) new_tag = (new_marker or {}).get("release_tag") or (new_marker or {}).get("tag") + # Pinned install must land on that exact release; a same-repo mismatch + # means the pin was ignored (Vulkan/Intel reroute to another repo is fine). + if ( + pin_release_tag + and new_tag + and (new_marker or {}).get("published_repo") == repo + and new_tag != pin_release_tag + ): + raise RuntimeError(f"pinned release {pin_release_tag} but installer produced {new_tag}") + with _job_lock: _job.update( state = _JOB_SUCCESS, @@ -650,6 +671,13 @@ def start_update() -> dict: repo = marker.get("published_repo") or DEFAULT_PUBLISHED_REPO from_tag = marker.get("tag") or marker.get("release_tag") asset = marker.get("asset") + # Install exactly the release the banner offered: the installer's own + # "latest" is commit-date ordered and can lag the published_at pick + # above, reinstalling the current build in a loop (the #6219 class). + # Not on macOS, which needs the older-release walk-back a pin disables + # (skipping too-new prebuilts); elsewhere an unusable latest now fails + # the job loudly (retryable) instead of walking back. + pin_release_tag = None if sys.platform == "darwin" else status.get("latest_tag") else: # Source build / custom path: only proceed when the same detection logic # would offer the update (prebuilt exists, install is behind, root is @@ -677,6 +705,9 @@ def start_update() -> dict: repo = (res or {}).get("repo") or DEFAULT_PUBLISHED_REPO from_tag = None asset = (res or {}).get("asset") + # No pin: source-build detection resolves via --resolve-prebuilt latest, + # the same resolver the unpinned apply uses, so the two already agree. + pin_release_tag = None if install_dir is None: return { @@ -704,7 +735,7 @@ def start_update() -> dict: thread = threading.Thread( target = _run_update, - args = (install_dir, repo, asset, script), + args = (install_dir, repo, asset, script, pin_release_tag), name = "llama-cpp-update", daemon = True, )