From cf43072a096f39b96a9cf9bf0dac45e1765d046b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 14 Jun 2026 22:23:40 -0700 Subject: [PATCH 1/6] Studio: route AMD ROCm llama.cpp installs and updates by GPU class The fork (unslothai/llama.cpp) ships per-gfx ROCm bundles for the consumer and APU arches (gfx103X/110X/120X/1150/1151) but not the CDNA data-center cards (MI100 gfx908, MI200 gfx90a), which only lemonade publishes. - Data-center GPUs: detect gfx908/gfx90a and serve them from lemonade at install time, recording the lemonade repo in the marker so updates re-check and re-install from lemonade. Lemonade assets stay hash-exempt (functional validation only); UNSLOTH_DISABLE_LEMONADE_ROCM opts out. - Consumer GPUs: a legacy lemonade install (pre-#6225) now updates from the fork, whose per-gfx bundles match what lemonade served. effective_published_repo redirects a source=lemonade marker to the fork unless it already recorded the lemonade repo (the data-center case), so the two paths do not collide. No change for NVIDIA, non-AMD, macOS, CPU, or consumer fork installs. --- studio/backend/tests/test_llama_cpp_update.py | 95 ++++++++++++++++ studio/backend/utils/llama_cpp_freshness.py | 23 +++- studio/backend/utils/llama_cpp_update.py | 5 +- studio/install_llama_prebuilt.py | 101 +++++++++++++++++- tests/studio/install/test_rocm_support.py | 71 ++++++++++++ 5 files changed, 291 insertions(+), 4 deletions(-) diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index 326b3dc6aa..44e8ab249a 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -83,6 +83,7 @@ def _write_install( repo: str = "unslothai/llama.cpp", asset: str | None = None, release_tag: str | None = None, + source: str | None = None, ) -> str: """Create a fake prebuilt install tree and return the llama-server path. @@ -104,6 +105,8 @@ def _write_install( } if asset is not None: marker["asset"] = asset + if source is not None: + marker["source"] = source (dir_ / MARKER).write_text(json.dumps(marker)) return str(binary) @@ -314,6 +317,98 @@ def test_status_up_to_date(monkeypatch, tmp_path): assert st["update_available"] is False +def test_status_lemonade_marker_compares_against_fork(monkeypatch, tmp_path): + # A pre-#6225 lemonade install whose marker recorded ggml-org (Adrenalin-only + # Windows) must compare against the fork's latest, not ggml-org's, so it + # migrates onto the fork's per-gfx bundle. + binary = _write_install( + tmp_path, + "b9518", + repo = "ggml-org/llama.cpp", + asset = "llama-b1292-windows-rocm-gfx1151-x64.zip", + source = "lemonade", + ) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50") + st = upd.get_update_status(force_refresh = True) + assert st["update_available"] is True + assert st["published_repo"] == "unslothai/llama.cpp" # not the recorded ggml-org + assert st["latest_tag"] == "b9632-mix-2d6bd50" + + +def test_start_update_lemonade_marker_targets_fork(monkeypatch, tmp_path): + # start_update on the same marker re-installs from the fork with the gfx + # recovered from the lemonade asset name. + install_dir = tmp_path / "llama.cpp" + binary = _write_install( + install_dir, + "b9518", + repo = "ggml-org/llama.cpp", + asset = "llama-b1292-windows-rocm-gfx1151-x64.zip", + source = "lemonade", + ) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50") + captured = {} + _patch_installer_popen(monkeypatch, on_start = lambda cmd: captured.update(cmd = cmd)) + res = upd.start_update() + assert res["started"] is True, res + deadline = time.time() + 10 + while time.time() < deadline: + if upd.get_update_status()["job"]["state"] in ("success", "error"): + break + time.sleep(0.05) + cmd = captured["cmd"] + assert "--published-repo" in cmd and "unslothai/llama.cpp" in cmd + assert "ggml-org/llama.cpp" not in cmd + assert cmd[cmd.index("--rocm-gfx") + 1] == "gfx1151" + + +def test_status_datacenter_lemonade_marker_stays_on_lemonade(monkeypatch, tmp_path): + # A data-center install (gfx908/gfx90a) recorded the lemonade repo; the fork + # has no such bundle, so its update must keep checking lemonade. + binary = _write_install( + tmp_path, + "b1292", + repo = "lemonade-sdk/llamacpp-rocm", + asset = "llama-b1292-ubuntu-rocm-gfx908-x64.zip", + source = "lemonade", + ) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b1300") + st = upd.get_update_status(force_refresh = True) + assert st["published_repo"] == "lemonade-sdk/llamacpp-rocm" + assert st["latest_tag"] == "b1300" + assert st["update_available"] is True + + +def test_start_update_datacenter_marker_targets_lemonade(monkeypatch, tmp_path): + install_dir = tmp_path / "llama.cpp" + binary = _write_install( + install_dir, + "b1292", + repo = "lemonade-sdk/llamacpp-rocm", + asset = "llama-b1292-ubuntu-rocm-gfx908-x64.zip", + source = "lemonade", + ) + monkeypatch.setattr(upd, "_find_binary", lambda: binary) + monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") + monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b1300") + captured = {} + _patch_installer_popen(monkeypatch, on_start = lambda cmd: captured.update(cmd = cmd)) + res = upd.start_update() + assert res["started"] is True, res + deadline = time.time() + 10 + while time.time() < deadline: + if upd.get_update_status()["job"]["state"] in ("success", "error"): + break + time.sleep(0.05) + cmd = captured["cmd"] + assert "--published-repo" in cmd and "lemonade-sdk/llamacpp-rocm" in cmd + assert cmd[cmd.index("--rocm-gfx") + 1] == "gfx908" + + def test_start_update_no_marker_no_prebuilt_refuses(monkeypatch, tmp_path): binary = tmp_path / "llama-server" binary.write_text("stub") # no marker diff --git a/studio/backend/utils/llama_cpp_freshness.py b/studio/backend/utils/llama_cpp_freshness.py index 87d0d2ec01..1244790794 100644 --- a/studio/backend/utils/llama_cpp_freshness.py +++ b/studio/backend/utils/llama_cpp_freshness.py @@ -31,6 +31,11 @@ _RELEASE_CACHE_TTL_SECONDS = 24 * 60 * 60 _INSTALL_MARKER_NAME = "UNSLOTH_PREBUILT_INFO.json" +# The fork now ships the per-gfx ROCm bundles lemonade used to (same gfx +# coverage), so lemonade installs update from the fork. See effective_published_repo. +DEFAULT_PUBLISHED_REPO = "unslothai/llama.cpp" +LEMONADE_ROCM_REPO = "lemonade-sdk/llamacpp-rocm" + _marker_cache: dict[str, Optional[dict]] = {} _release_memo: dict[str, tuple[float, Optional[str]]] = {} @@ -72,6 +77,21 @@ def read_install_marker(binary_path: Optional[str]) -> Optional[dict]: return marker +def effective_published_repo(marker: Optional[dict]) -> Optional[str]: + """Repo whose releases supersede this install. A pre-#6225 consumer lemonade + install recorded a non-fork published_repo (the fork for tooling hosts, but + ggml-org for Adrenalin-only Windows), yet the fork now ships those per-gfx + ROCm bundles; route its updates to the fork. Data-center installs (gfx908/ + gfx90a) recorded the lemonade repo and keep updating from it (the fork has no + such bundle). Non-lemonade markers keep their published_repo.""" + if not marker: + return None + repo = marker.get("published_repo") + if (marker.get("source") or "").lower() == "lemonade" and repo != LEMONADE_ROCM_REPO: + return DEFAULT_PUBLISHED_REPO + return repo + + def _cache_path_for(repo: str) -> Path: safe = repo.replace("/", "__") return _cache_dir() / f"{safe}.json" @@ -260,7 +280,8 @@ def check_prebuilt_freshness( # full "release_tag" -- deliberately opposite fallbacks. out["installed_tag"] = marker.get("tag") or marker.get("release_tag") out["installed_at_utc"] = marker.get("installed_at_utc") - out["published_repo"] = marker.get("published_repo") + # Lemonade installs compare against (and later re-install from) the fork. + out["published_repo"] = effective_published_repo(marker) # The marker records both a normalized base tag ("tag", e.g. b9596) and the # full release tag ("release_tag", e.g. b9596-mix-). Compare against the diff --git a/studio/backend/utils/llama_cpp_update.py b/studio/backend/utils/llama_cpp_update.py index 3518247d7e..f24444ebad 100644 --- a/studio/backend/utils/llama_cpp_update.py +++ b/studio/backend/utils/llama_cpp_update.py @@ -36,6 +36,7 @@ import structlog from utils.llama_cpp_freshness import ( _INSTALL_MARKER_NAME, check_prebuilt_freshness, + effective_published_repo, latest_published_release, parse_base_build, read_install_marker, @@ -294,7 +295,7 @@ def get_update_status(*, force_refresh: bool = False) -> dict: if src is not None: return src - repo = (marker or {}).get("published_repo") or DEFAULT_PUBLISHED_REPO + repo = effective_published_repo(marker) or DEFAULT_PUBLISHED_REPO if force_refresh and repo: # Prime the cache so the freshness read below sees the newest tag. @@ -505,7 +506,7 @@ def start_update() -> dict: "job": status["job"], } install_dir = _install_dir_for(binary) - repo = marker.get("published_repo") or DEFAULT_PUBLISHED_REPO + repo = effective_published_repo(marker) or DEFAULT_PUBLISHED_REPO from_tag = marker.get("tag") or marker.get("release_tag") asset = marker.get("asset") else: diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index b33d0b6325..6c5c674a28 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -139,6 +139,11 @@ DEFAULT_PUBLISHED_SHA256_ASSET = os.environ.get( UPSTREAM_REPO = "ggml-org/llama.cpp" UPSTREAM_RELEASES_API = f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/latest" +# CDNA data-center arches lemonade ships ROCm bundles for that the fork's per-gfx +# bundles do not (MI100 / MI200). Consumer/APU arches are served by the fork. +LEMONADE_ROCM_REPO = "lemonade-sdk/llamacpp-rocm" +_LEMONADE_DATACENTER_ARCHES = ("gfx908", "gfx90a") + TEST_MODEL_URL = "https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf" TEST_MODEL_SHA256 = "270cba1bd5109f42d03350f60406024560464db173c0e387d91f0426d3bd256d" @@ -1588,6 +1593,89 @@ def pinned_macos_release_tag(host: HostInfo, repo: str) -> str | None: return _PINNED_MACOS_FALLBACK_TAG +def _lemonade_datacenter_gfx(gfx_target: str | None) -> str | None: + """The host gfx if it is a data-center arch only lemonade covers, else None.""" + gfx = (gfx_target or "").lower().strip() + return gfx if gfx in _LEMONADE_DATACENTER_ARCHES else None + + +def _lemonade_release_api_for(llama_tag: str) -> str: + """Lemonade release API URL. Lemonade uses its own tag series, so a pinned + fork/ggml-org tag will 404 here; only 'latest' or a lemonade tag resolve.""" + tag = (llama_tag or "").strip() + if not tag or tag.lower() == "latest": + return f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/latest" + return ( + f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/tags/" + f"{urllib.parse.quote(tag, safe = '')}" + ) + + +def resolve_lemonade_rocm_choice( + host: HostInfo, os_prefix: str, install_kind: str, llama_tag: str = "latest" +) -> AssetChoice | None: + """Lemonade ROCm bundle for a data-center AMD GPU (gfx908/gfx90a) the fork + does not ship. None for any other arch or on a fetch/asset/host miss. + Lemonade assets are not in the approved-hash manifest, so integrity is + functional validation only; UNSLOTH_DISABLE_LEMONADE_ROCM opts out.""" + gfx = _lemonade_datacenter_gfx(host.rocm_gfx_target) + if gfx is None: + return None + if os.environ.get("UNSLOTH_DISABLE_LEMONADE_ROCM", "").strip().lower() in ("1", "true", "yes"): + log("UNSLOTH_DISABLE_LEMONADE_ROCM is set; skipping lemonade-sdk prebuilt") + return None + try: + release = fetch_json(_lemonade_release_api_for(llama_tag)) + except Exception as exc: + log(f"Could not fetch {LEMONADE_ROCM_REPO} release ({exc}); skipping lemonade prebuilt") + return None + release_tag = release.get("tag_name") if isinstance(release, dict) else None + if not isinstance(release_tag, str) or not release_tag: + return None + asset_name = f"llama-{release_tag}-{os_prefix}-rocm-{gfx}-x64.zip" + asset_url = release_asset_map(release).get(asset_name) + # browser_download_url for a release asset is always github.com//download. + trusted = (asset_url or "").startswith( + f"https://github.com/{LEMONADE_ROCM_REPO}/releases/download/" + ) + if not asset_url or not trusted: + log(f"{LEMONADE_ROCM_REPO}@{release_tag} missing or untrusted {asset_name!r}; skipping") + return None + return AssetChoice( + repo = LEMONADE_ROCM_REPO, + tag = release_tag, + name = asset_name, + url = asset_url, + source_label = "lemonade", + install_kind = install_kind, + selection_log = [f"rocm_selection: data-center GPU {host.rocm_gfx_target} -> {asset_name}"], + ) + + +def _lemonade_release_plans( + llama_tag: str, host: HostInfo +) -> tuple[str, list[InstallReleasePlan]]: + """Single-attempt plan for a data-center AMD GPU, sourced from lemonade. The + marker records the lemonade repo/tag so updates re-check and re-install it.""" + requested_tag = normalized_requested_llama_tag(llama_tag) + os_prefix, install_kind = ( + ("windows", "windows-hip") if host.is_windows else ("ubuntu", "linux-rocm") + ) + choice = resolve_lemonade_rocm_choice(host, os_prefix, install_kind, llama_tag = requested_tag) + if choice is None: + raise PrebuiltFallback("no lemonade ROCm prebuilt for this data-center GPU") + plan = InstallReleasePlan( + requested_tag = requested_tag, + llama_tag = choice.tag, + release_tag = choice.tag, + attempts = [choice], + approved_checksums = synthetic_checksums_for_release( + LEMONADE_ROCM_REPO, choice.tag, choice.tag + ), + ) + return requested_tag, [plan] + + def resolve_simple_install_release_plans( llama_tag: str, host: HostInfo, @@ -1597,6 +1685,15 @@ def resolve_simple_install_release_plans( max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS, ) -> tuple[str, list[InstallReleasePlan]]: repo = published_repo or DEFAULT_PUBLISHED_REPO + # Data-center AMD GPUs (gfx908/gfx90a) are not in the fork's per-gfx bundles; + # serve them from lemonade. Catches the fork-routed fresh install and the + # lemonade-repo update re-install alike. + if ( + host.is_x86_64 + and not host.has_usable_nvidia + and (repo == LEMONADE_ROCM_REPO or _lemonade_datacenter_gfx(host.rocm_gfx_target)) + ): + return _lemonade_release_plans(llama_tag, host) # The fork (unslothai) ships a manifest describing every bundle's GPU/arch # coverage, so all fork hosts select from it. Upstream (ggml-org) ships no # manifest and is selected by asset filename in the loop below. @@ -6857,7 +6954,9 @@ def main() -> int: else: payload = { "prebuilt_available": True, - "repo": repo, + # The plan's repo, not the host default: a data-center GPU + # routed to the fork is actually served from lemonade. + "repo": plans[0].approved_checksums.repo or repo, "release_tag": plans[0].release_tag, "llama_tag": plans[0].llama_tag, "asset": choice.name, diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index a3c9555d0b..e287e310c3 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -330,6 +330,77 @@ class TestResolveUpstreamAssetChoice: resolve_upstream_asset_choice(host, LLAMA_TAG) +# TEST: install_llama_prebuilt.py -- data-center ROCm (lemonade) routing + + +def _lemonade_release(tag = "b1300"): + base = f"https://github.com/lemonade-sdk/llamacpp-rocm/releases/download/{tag}" + names = [ + f"llama-{tag}-ubuntu-rocm-gfx908-x64.zip", + f"llama-{tag}-ubuntu-rocm-gfx90a-x64.zip", + f"llama-{tag}-windows-rocm-gfx908-x64.zip", + ] + return { + "tag_name": tag, + "assets": [{"name": n, "browser_download_url": f"{base}/{n}"} for n in names], + } + + +class TestDataCenterLemonadeRouting: + """gfx908/gfx90a (MI100/MI200) are not in the fork's per-gfx bundles, so they + are served from lemonade at install and update time. Consumer arches and + NVIDIA hosts must not be routed here.""" + + @patch.object(prebuilt_mod, "fetch_json", return_value = _lemonade_release()) + def test_datacenter_linux_routes_to_lemonade(self, _mock): + host = rocm_host(rocm_gfx_target = "gfx908") + _tag, plans = prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" + ) + choice = plans[0].attempts[0] + assert choice.source_label == "lemonade" + assert choice.repo == prebuilt_mod.LEMONADE_ROCM_REPO + assert choice.install_kind == "linux-rocm" and "gfx908" in choice.name + # Marker repo (published_repo) is lemonade, so updates re-check lemonade. + assert plans[0].approved_checksums.repo == prebuilt_mod.LEMONADE_ROCM_REPO + + @patch.object(prebuilt_mod, "fetch_json", return_value = _lemonade_release()) + def test_update_via_lemonade_repo_routes_to_lemonade(self, _mock): + # The update path re-invokes with --published-repo lemonade + --rocm-gfx. + host = rocm_host(rocm_gfx_target = "gfx90a") + _tag, plans = prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.LEMONADE_ROCM_REPO, "" + ) + assert plans[0].attempts[0].name.endswith("ubuntu-rocm-gfx90a-x64.zip") + + def test_consumer_arch_not_routed_to_lemonade(self, monkeypatch): + # gfx1151 is a fork family: lemonade returns None and the dispatch must + # fall through to the fork manifest path, not lemonade. + host = rocm_host(rocm_gfx_target = "gfx1151") + assert prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") is None + sentinel = ("forkpath", []) + monkeypatch.setattr(prebuilt_mod, "_fork_manifest_release_plans", lambda *a, **k: sentinel) + assert prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" + ) == sentinel + + def test_nvidia_with_stray_gfx_not_routed_to_lemonade(self, monkeypatch): + host = nvidia_host(has_rocm = True, rocm_gfx_target = "gfx908") + sentinel = ("forkpath", []) + monkeypatch.setattr(prebuilt_mod, "_fork_manifest_release_plans", lambda *a, **k: sentinel) + assert prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" + ) == sentinel + + @patch.object(prebuilt_mod, "fetch_json") + def test_untrusted_asset_url_skipped(self, mock_fetch): + rel = _lemonade_release() + rel["assets"][0]["browser_download_url"] = "https://evil.example.com/x.zip" + mock_fetch.return_value = rel + host = rocm_host(rocm_gfx_target = "gfx908") + assert prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") is None + + # TEST: install_llama_prebuilt.py -- runtime_patterns_for_choice From ac5652be6b02d1f4f6fe7313b9f745f35c42db77 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sun, 14 Jun 2026 23:15:44 -0700 Subject: [PATCH 2/6] Studio: hydrate a real upstream source tree for data-center lemonade installs Review fix. The data-center plan set llama_tag to lemonade's own release counter (b1292), but validate_prebuilt_choice hydrates the ggml-org source tree by llama_tag. ggml-org has an unrelated ancient b1292 tag, so the install would overlay a 2026 lemonade binary onto a 2023 source tree (or 404 for lemonade tags with no ggml-org counterpart). Resolve llama_tag to the upstream latest for hydration and keep release_tag as the lemonade tag for update tracking. Display the lemonade release_tag in the freshness banner so installed and latest stay in one series. --- studio/backend/tests/test_llama_cpp_update.py | 10 +++++++--- studio/backend/utils/llama_cpp_freshness.py | 9 +++++++-- studio/install_llama_prebuilt.py | 14 ++++++++++---- tests/studio/install/test_rocm_support.py | 16 +++++++++++++--- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index 44e8ab249a..1db05b2b42 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -367,18 +367,21 @@ def test_start_update_lemonade_marker_targets_fork(monkeypatch, tmp_path): def test_status_datacenter_lemonade_marker_stays_on_lemonade(monkeypatch, tmp_path): # A data-center install (gfx908/gfx90a) recorded the lemonade repo; the fork - # has no such bundle, so its update must keep checking lemonade. + # has no such bundle, so its update must keep checking lemonade. The marker's + # tag is the upstream source build; release_tag is lemonade's own counter. binary = _write_install( tmp_path, - "b1292", + "b9637", repo = "lemonade-sdk/llamacpp-rocm", asset = "llama-b1292-ubuntu-rocm-gfx908-x64.zip", + release_tag = "b1292", source = "lemonade", ) monkeypatch.setattr(upd, "_find_binary", lambda: binary) monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b1300") st = upd.get_update_status(force_refresh = True) assert st["published_repo"] == "lemonade-sdk/llamacpp-rocm" + assert st["installed_tag"] == "b1292" # lemonade series, not the upstream tag assert st["latest_tag"] == "b1300" assert st["update_available"] is True @@ -387,9 +390,10 @@ def test_start_update_datacenter_marker_targets_lemonade(monkeypatch, tmp_path): install_dir = tmp_path / "llama.cpp" binary = _write_install( install_dir, - "b1292", + "b9637", repo = "lemonade-sdk/llamacpp-rocm", asset = "llama-b1292-ubuntu-rocm-gfx908-x64.zip", + release_tag = "b1292", source = "lemonade", ) monkeypatch.setattr(upd, "_find_binary", lambda: binary) diff --git a/studio/backend/utils/llama_cpp_freshness.py b/studio/backend/utils/llama_cpp_freshness.py index 1244790794..a5ab59c7dd 100644 --- a/studio/backend/utils/llama_cpp_freshness.py +++ b/studio/backend/utils/llama_cpp_freshness.py @@ -277,8 +277,13 @@ def check_prebuilt_freshness( return out out["has_marker"] = True # Display prefers the normalized base ("tag"); comparison below prefers the - # full "release_tag" -- deliberately opposite fallbacks. - out["installed_tag"] = marker.get("tag") or marker.get("release_tag") + # full "release_tag" -- deliberately opposite fallbacks. A lemonade install's + # "tag" is the upstream build it was made from, but it tracks lemonade's own + # release counter, so show release_tag to keep installed/latest one series. + if (marker.get("source") or "").lower() == "lemonade": + out["installed_tag"] = marker.get("release_tag") or marker.get("tag") + else: + out["installed_tag"] = marker.get("tag") or marker.get("release_tag") out["installed_at_utc"] = marker.get("installed_at_utc") # Lemonade installs compare against (and later re-install from) the fork. out["published_repo"] = effective_published_repo(marker) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 6c5c674a28..a5020e7740 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1655,8 +1655,10 @@ def resolve_lemonade_rocm_choice( def _lemonade_release_plans( llama_tag: str, host: HostInfo ) -> tuple[str, list[InstallReleasePlan]]: - """Single-attempt plan for a data-center AMD GPU, sourced from lemonade. The - marker records the lemonade repo/tag so updates re-check and re-install it.""" + """Single-attempt plan for a data-center AMD GPU, sourced from lemonade. + release_tag is lemonade's own counter so updates compare against lemonade; + llama_tag is a real upstream tag because the source tree is hydrated from + ggml-org by it (lemonade's counter is not a ggml-org ref).""" requested_tag = normalized_requested_llama_tag(llama_tag) os_prefix, install_kind = ( ("windows", "windows-hip") if host.is_windows else ("ubuntu", "linux-rocm") @@ -1664,13 +1666,17 @@ def _lemonade_release_plans( choice = resolve_lemonade_rocm_choice(host, os_prefix, install_kind, llama_tag = requested_tag) if choice is None: raise PrebuiltFallback("no lemonade ROCm prebuilt for this data-center GPU") + try: + upstream_tag = latest_upstream_release_tag() + except Exception as exc: + raise PrebuiltFallback(f"could not resolve upstream source tag for lemonade install: {exc}") plan = InstallReleasePlan( requested_tag = requested_tag, - llama_tag = choice.tag, + llama_tag = upstream_tag, release_tag = choice.tag, attempts = [choice], approved_checksums = synthetic_checksums_for_release( - LEMONADE_ROCM_REPO, choice.tag, choice.tag + LEMONADE_ROCM_REPO, choice.tag, upstream_tag ), ) return requested_tag, [plan] diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index e287e310c3..cb51e4520d 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -346,12 +346,19 @@ def _lemonade_release(tag = "b1300"): } +def _fetch_router(url): + # Lemonade lookup vs the upstream-latest lookup the plan does for source. + if "lemonade-sdk" in url: + return _lemonade_release() + return {"tag_name": "b9637"} # ggml-org upstream latest (for source hydration) + + class TestDataCenterLemonadeRouting: """gfx908/gfx90a (MI100/MI200) are not in the fork's per-gfx bundles, so they are served from lemonade at install and update time. Consumer arches and NVIDIA hosts must not be routed here.""" - @patch.object(prebuilt_mod, "fetch_json", return_value = _lemonade_release()) + @patch.object(prebuilt_mod, "fetch_json", side_effect = _fetch_router) def test_datacenter_linux_routes_to_lemonade(self, _mock): host = rocm_host(rocm_gfx_target = "gfx908") _tag, plans = prebuilt_mod.resolve_simple_install_release_plans( @@ -361,10 +368,13 @@ class TestDataCenterLemonadeRouting: assert choice.source_label == "lemonade" assert choice.repo == prebuilt_mod.LEMONADE_ROCM_REPO assert choice.install_kind == "linux-rocm" and "gfx908" in choice.name - # Marker repo (published_repo) is lemonade, so updates re-check lemonade. + # published_repo is lemonade (updates re-check lemonade); release_tag is + # lemonade's counter; llama_tag is a real upstream tag for source hydration. assert plans[0].approved_checksums.repo == prebuilt_mod.LEMONADE_ROCM_REPO + assert plans[0].release_tag == "b1300" + assert plans[0].llama_tag == "b9637" - @patch.object(prebuilt_mod, "fetch_json", return_value = _lemonade_release()) + @patch.object(prebuilt_mod, "fetch_json", side_effect = _fetch_router) def test_update_via_lemonade_repo_routes_to_lemonade(self, _mock): # The update path re-invokes with --published-repo lemonade + --rocm-gfx. host = rocm_host(rocm_gfx_target = "gfx90a") From 4a02cedf60f50b5a2a269b2f71c19c156d651108 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 06:16:39 +0000 Subject: [PATCH 3/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/tests/test_llama_cpp_update.py | 8 ++++++-- studio/install_llama_prebuilt.py | 9 +++++---- tests/studio/install/test_rocm_support.py | 18 ++++++++++++------ 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/studio/backend/tests/test_llama_cpp_update.py b/studio/backend/tests/test_llama_cpp_update.py index 1db05b2b42..522466a9f4 100644 --- a/studio/backend/tests/test_llama_cpp_update.py +++ b/studio/backend/tests/test_llama_cpp_update.py @@ -329,7 +329,9 @@ def test_status_lemonade_marker_compares_against_fork(monkeypatch, tmp_path): source = "lemonade", ) monkeypatch.setattr(upd, "_find_binary", lambda: binary) - monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50") + monkeypatch.setattr( + freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50" + ) st = upd.get_update_status(force_refresh = True) assert st["update_available"] is True assert st["published_repo"] == "unslothai/llama.cpp" # not the recorded ggml-org @@ -349,7 +351,9 @@ def test_start_update_lemonade_marker_targets_fork(monkeypatch, tmp_path): ) monkeypatch.setattr(upd, "_find_binary", lambda: binary) monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py") - monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50") + monkeypatch.setattr( + freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50" + ) captured = {} _patch_installer_popen(monkeypatch, on_start = lambda cmd: captured.update(cmd = cmd)) res = upd.start_update() diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index a5020e7740..dc2de31773 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1612,7 +1612,10 @@ def _lemonade_release_api_for(llama_tag: str) -> str: def resolve_lemonade_rocm_choice( - host: HostInfo, os_prefix: str, install_kind: str, llama_tag: str = "latest" + host: HostInfo, + os_prefix: str, + install_kind: str, + llama_tag: str = "latest", ) -> AssetChoice | None: """Lemonade ROCm bundle for a data-center AMD GPU (gfx908/gfx90a) the fork does not ship. None for any other arch or on a fetch/asset/host miss. @@ -1652,9 +1655,7 @@ def resolve_lemonade_rocm_choice( ) -def _lemonade_release_plans( - llama_tag: str, host: HostInfo -) -> tuple[str, list[InstallReleasePlan]]: +def _lemonade_release_plans(llama_tag: str, host: HostInfo) -> tuple[str, list[InstallReleasePlan]]: """Single-attempt plan for a data-center AMD GPU, sourced from lemonade. release_tag is lemonade's own counter so updates compare against lemonade; llama_tag is a real upstream tag because the source tree is hydrated from diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index cb51e4520d..66d8261941 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -390,17 +390,23 @@ class TestDataCenterLemonadeRouting: assert prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") is None sentinel = ("forkpath", []) monkeypatch.setattr(prebuilt_mod, "_fork_manifest_release_plans", lambda *a, **k: sentinel) - assert prebuilt_mod.resolve_simple_install_release_plans( - "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" - ) == sentinel + assert ( + prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" + ) + == sentinel + ) def test_nvidia_with_stray_gfx_not_routed_to_lemonade(self, monkeypatch): host = nvidia_host(has_rocm = True, rocm_gfx_target = "gfx908") sentinel = ("forkpath", []) monkeypatch.setattr(prebuilt_mod, "_fork_manifest_release_plans", lambda *a, **k: sentinel) - assert prebuilt_mod.resolve_simple_install_release_plans( - "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" - ) == sentinel + assert ( + prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" + ) + == sentinel + ) @patch.object(prebuilt_mod, "fetch_json") def test_untrusted_asset_url_skipped(self, mock_fetch): From 10d7ac5eb20b728bb8fa3cf58e6156f3c2ce0ce3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 15 Jun 2026 06:00:40 -0700 Subject: [PATCH 4/6] Studio: align data-center lemonade selection with the time-ordered resolver Match the lemonade ROCm path to how the fork/upstream resolver and the update banner already pick releases, so install and update agree: - Scan recent lemonade releases newest-by-publish-time instead of calling /releases/latest, which sorts by commit date and can lag. The banner's latest_published_release() already does this, so they no longer disagree and leave MI100/MI200 hosts with a sticky banner or a downgrade. - Walk back past a newest release that is missing this GPU's asset to an older complete one, mirroring the fork resolver's release fallback, instead of dropping straight to a HIP source build. - Honor a pinned --published-release-tag for gfx908/gfx90a so the pin/rollback path works the same as the fork/upstream branches. - effective_published_repo: guard against a non-dict marker or non-string source so corrupt on-disk markers cannot crash update-status polling. Tests cover walk-back, publish-time ordering, and pin selection. --- studio/backend/utils/llama_cpp_freshness.py | 6 +- studio/install_llama_prebuilt.py | 105 ++++++++++++-------- tests/studio/install/test_rocm_support.py | 59 +++++++++-- 3 files changed, 116 insertions(+), 54 deletions(-) diff --git a/studio/backend/utils/llama_cpp_freshness.py b/studio/backend/utils/llama_cpp_freshness.py index a5ab59c7dd..810e8dc492 100644 --- a/studio/backend/utils/llama_cpp_freshness.py +++ b/studio/backend/utils/llama_cpp_freshness.py @@ -84,10 +84,12 @@ def effective_published_repo(marker: Optional[dict]) -> Optional[str]: ROCm bundles; route its updates to the fork. Data-center installs (gfx908/ gfx90a) recorded the lemonade repo and keep updating from it (the fork has no such bundle). Non-lemonade markers keep their published_repo.""" - if not marker: + if not isinstance(marker, dict): return None repo = marker.get("published_repo") - if (marker.get("source") or "").lower() == "lemonade" and repo != LEMONADE_ROCM_REPO: + source = marker.get("source") + source = source.lower() if isinstance(source, str) else "" + if source == "lemonade" and repo != LEMONADE_ROCM_REPO: return DEFAULT_PUBLISHED_REPO return repo diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index dc2de31773..ee86a985b1 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1599,16 +1599,34 @@ def _lemonade_datacenter_gfx(gfx_target: str | None) -> str | None: return gfx if gfx in _LEMONADE_DATACENTER_ARCHES else None -def _lemonade_release_api_for(llama_tag: str) -> str: - """Lemonade release API URL. Lemonade uses its own tag series, so a pinned - fork/ggml-org tag will 404 here; only 'latest' or a lemonade tag resolve.""" +def _lemonade_releases_newest_first(llama_tag: str) -> list[dict[str, Any]]: + """Lemonade releases to try, newest-published first. A pinned tag yields just + that release; 'latest' scans the recent list ordered by publish time -- the + same ordering the update banner uses, so detection and install agree (the + /releases/latest pointer sorts by commit date and can lag). Lemonade uses its + own tag series, so a pinned fork/ggml-org tag simply 404s to an empty list.""" tag = (llama_tag or "").strip() - if not tag or tag.lower() == "latest": - return f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/latest" - return ( - f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/tags/" - f"{urllib.parse.quote(tag, safe = '')}" - ) + api = f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases" + try: + if tag and tag.lower() != "latest": + return [fetch_json(f"{api}/tags/{urllib.parse.quote(tag, safe = '')}")] + payload = fetch_json(f"{api}?per_page=30") + except Exception as exc: + log(f"Could not fetch {LEMONADE_ROCM_REPO} releases ({exc}); skipping lemonade prebuilt") + return [] + if not isinstance(payload, list): + return [] + rels = [ + r + for r in payload + if isinstance(r, dict) + and not r.get("draft") + and not r.get("prerelease") + and isinstance(r.get("tag_name"), str) + and r.get("tag_name") + ] + rels.sort(key = lambda r: r.get("published_at") or "", reverse = True) + return rels def resolve_lemonade_rocm_choice( @@ -1618,53 +1636,56 @@ def resolve_lemonade_rocm_choice( llama_tag: str = "latest", ) -> AssetChoice | None: """Lemonade ROCm bundle for a data-center AMD GPU (gfx908/gfx90a) the fork - does not ship. None for any other arch or on a fetch/asset/host miss. - Lemonade assets are not in the approved-hash manifest, so integrity is - functional validation only; UNSLOTH_DISABLE_LEMONADE_ROCM opts out.""" + does not ship. Walks recent lemonade releases newest-first and takes the + first carrying this GPU's asset, so a partial newest nightly falls back to an + older complete one (mirrors the fork resolver). None for any other arch, an + opt-out, or when no scanned release has the asset. Lemonade assets are not in + the approved-hash manifest, so integrity is functional validation only.""" gfx = _lemonade_datacenter_gfx(host.rocm_gfx_target) if gfx is None: return None if os.environ.get("UNSLOTH_DISABLE_LEMONADE_ROCM", "").strip().lower() in ("1", "true", "yes"): log("UNSLOTH_DISABLE_LEMONADE_ROCM is set; skipping lemonade-sdk prebuilt") return None - try: - release = fetch_json(_lemonade_release_api_for(llama_tag)) - except Exception as exc: - log(f"Could not fetch {LEMONADE_ROCM_REPO} release ({exc}); skipping lemonade prebuilt") - return None - release_tag = release.get("tag_name") if isinstance(release, dict) else None - if not isinstance(release_tag, str) or not release_tag: - return None - asset_name = f"llama-{release_tag}-{os_prefix}-rocm-{gfx}-x64.zip" - asset_url = release_asset_map(release).get(asset_name) - # browser_download_url for a release asset is always github.com//download. - trusted = (asset_url or "").startswith( - f"https://github.com/{LEMONADE_ROCM_REPO}/releases/download/" - ) - if not asset_url or not trusted: - log(f"{LEMONADE_ROCM_REPO}@{release_tag} missing or untrusted {asset_name!r}; skipping") - return None - return AssetChoice( - repo = LEMONADE_ROCM_REPO, - tag = release_tag, - name = asset_name, - url = asset_url, - source_label = "lemonade", - install_kind = install_kind, - selection_log = [f"rocm_selection: data-center GPU {host.rocm_gfx_target} -> {asset_name}"], - ) + for release in _lemonade_releases_newest_first(llama_tag): + release_tag = release.get("tag_name") if isinstance(release, dict) else None + if not isinstance(release_tag, str) or not release_tag: + continue + asset_name = f"llama-{release_tag}-{os_prefix}-rocm-{gfx}-x64.zip" + asset_url = release_asset_map(release).get(asset_name) + # browser_download_url for a release asset is always github.com//download. + if asset_url and asset_url.startswith( + f"https://github.com/{LEMONADE_ROCM_REPO}/releases/download/" + ): + return AssetChoice( + repo = LEMONADE_ROCM_REPO, + tag = release_tag, + name = asset_name, + url = asset_url, + source_label = "lemonade", + install_kind = install_kind, + selection_log = [ + f"rocm_selection: data-center GPU {host.rocm_gfx_target} -> {asset_name}" + ], + ) + log(f"no recent {LEMONADE_ROCM_REPO} release carries a {gfx} asset; skipping lemonade prebuilt") + return None -def _lemonade_release_plans(llama_tag: str, host: HostInfo) -> tuple[str, list[InstallReleasePlan]]: +def _lemonade_release_plans( + llama_tag: str, host: HostInfo, published_release_tag: str = "" +) -> tuple[str, list[InstallReleasePlan]]: """Single-attempt plan for a data-center AMD GPU, sourced from lemonade. release_tag is lemonade's own counter so updates compare against lemonade; llama_tag is a real upstream tag because the source tree is hydrated from - ggml-org by it (lemonade's counter is not a ggml-org ref).""" + ggml-org by it (lemonade's counter is not a ggml-org ref). A pinned + published_release_tag selects a specific lemonade release for rollback.""" requested_tag = normalized_requested_llama_tag(llama_tag) + lemonade_tag = published_release_tag.strip() or requested_tag os_prefix, install_kind = ( ("windows", "windows-hip") if host.is_windows else ("ubuntu", "linux-rocm") ) - choice = resolve_lemonade_rocm_choice(host, os_prefix, install_kind, llama_tag = requested_tag) + choice = resolve_lemonade_rocm_choice(host, os_prefix, install_kind, llama_tag = lemonade_tag) if choice is None: raise PrebuiltFallback("no lemonade ROCm prebuilt for this data-center GPU") try: @@ -1700,7 +1721,7 @@ def resolve_simple_install_release_plans( and not host.has_usable_nvidia and (repo == LEMONADE_ROCM_REPO or _lemonade_datacenter_gfx(host.rocm_gfx_target)) ): - return _lemonade_release_plans(llama_tag, host) + return _lemonade_release_plans(llama_tag, host, published_release_tag) # The fork (unslothai) ships a manifest describing every bundle's GPU/arch # coverage, so all fork hosts select from it. Upstream (ggml-org) ships no # manifest and is selected by asset filename in the loop below. diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 66d8261941..338cbc1cca 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -333,23 +333,22 @@ class TestResolveUpstreamAssetChoice: # TEST: install_llama_prebuilt.py -- data-center ROCm (lemonade) routing -def _lemonade_release(tag = "b1300"): +def _lemonade_release(tag = "b1300", published_at = "2026-01-01T00:00:00Z", gfxs = None): base = f"https://github.com/lemonade-sdk/llamacpp-rocm/releases/download/{tag}" - names = [ - f"llama-{tag}-ubuntu-rocm-gfx908-x64.zip", - f"llama-{tag}-ubuntu-rocm-gfx90a-x64.zip", - f"llama-{tag}-windows-rocm-gfx908-x64.zip", - ] + arches = gfxs if gfxs is not None else ["gfx908", "gfx90a"] + names = [f"llama-{tag}-ubuntu-rocm-{g}-x64.zip" for g in arches] + names += [f"llama-{tag}-windows-rocm-{g}-x64.zip" for g in arches] return { "tag_name": tag, + "published_at": published_at, "assets": [{"name": n, "browser_download_url": f"{base}/{n}"} for n in names], } def _fetch_router(url): - # Lemonade lookup vs the upstream-latest lookup the plan does for source. + # Lemonade list lookup ('latest') vs upstream-latest lookup for source. if "lemonade-sdk" in url: - return _lemonade_release() + return [_lemonade_release()] if "?per_page" in url else _lemonade_release() return {"tag_name": "b9637"} # ggml-org upstream latest (for source hydration) @@ -410,12 +409,52 @@ class TestDataCenterLemonadeRouting: @patch.object(prebuilt_mod, "fetch_json") def test_untrusted_asset_url_skipped(self, mock_fetch): - rel = _lemonade_release() + rel = _lemonade_release(gfxs = ["gfx908"]) rel["assets"][0]["browser_download_url"] = "https://evil.example.com/x.zip" - mock_fetch.return_value = rel + mock_fetch.return_value = [rel] host = rocm_host(rocm_gfx_target = "gfx908") assert prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") is None + @patch.object(prebuilt_mod, "fetch_json") + def test_walk_back_past_partial_newest(self, mock_fetch): + # Newest release has no gfx908 asset yet -> fall back to the older one + # that does, mirroring the fork resolver's release walk-back. + newest = _lemonade_release(tag = "b1305", published_at = "2026-02-01T00:00:00Z", gfxs = []) + older = _lemonade_release(tag = "b1300", published_at = "2026-01-15T00:00:00Z") + mock_fetch.return_value = [newest, older] + host = rocm_host(rocm_gfx_target = "gfx908") + choice = prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") + assert choice is not None and choice.tag == "b1300" + + @patch.object(prebuilt_mod, "fetch_json") + def test_newest_by_publish_time_wins(self, mock_fetch): + # List order is not release order; pick the newest published_at. + mock_fetch.return_value = [ + _lemonade_release(tag = "b1290", published_at = "2026-01-01T00:00:00Z"), + _lemonade_release(tag = "b1301", published_at = "2026-03-01T00:00:00Z"), + ] + host = rocm_host(rocm_gfx_target = "gfx908") + choice = prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") + assert choice is not None and choice.tag == "b1301" + + @patch.object(prebuilt_mod, "fetch_json") + def test_pinned_release_tag_selected(self, mock_fetch): + # A pinned lemonade release_tag resolves that exact release (tags endpoint), + # while source still hydrates from a real upstream tag. + def _router(url): + if "lemonade-sdk" in url and "/tags/b1295" in url: + return _lemonade_release(tag = "b1295") + if "lemonade-sdk" in url: + return [_lemonade_release(tag = "b1301")] # would win if pin ignored + return {"tag_name": "b9637"} + mock_fetch.side_effect = _router + host = rocm_host(rocm_gfx_target = "gfx908") + _tag, plans = prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.LEMONADE_ROCM_REPO, "b1295" + ) + assert plans[0].release_tag == "b1295" + assert plans[0].llama_tag == "b9637" + # TEST: install_llama_prebuilt.py -- runtime_patterns_for_choice From ffd7932860fa0a6a8b9fbc39b7db9c52a21bf371 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:03:27 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/install_llama_prebuilt.py | 4 +++- tests/studio/install/test_rocm_support.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index ee86a985b1..0345bf6622 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1673,7 +1673,9 @@ def resolve_lemonade_rocm_choice( def _lemonade_release_plans( - llama_tag: str, host: HostInfo, published_release_tag: str = "" + llama_tag: str, + host: HostInfo, + published_release_tag: str = "", ) -> tuple[str, list[InstallReleasePlan]]: """Single-attempt plan for a data-center AMD GPU, sourced from lemonade. release_tag is lemonade's own counter so updates compare against lemonade; diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 338cbc1cca..8a6c64052e 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -333,7 +333,11 @@ class TestResolveUpstreamAssetChoice: # TEST: install_llama_prebuilt.py -- data-center ROCm (lemonade) routing -def _lemonade_release(tag = "b1300", published_at = "2026-01-01T00:00:00Z", gfxs = None): +def _lemonade_release( + tag = "b1300", + published_at = "2026-01-01T00:00:00Z", + gfxs = None, +): base = f"https://github.com/lemonade-sdk/llamacpp-rocm/releases/download/{tag}" arches = gfxs if gfxs is not None else ["gfx908", "gfx90a"] names = [f"llama-{tag}-ubuntu-rocm-{g}-x64.zip" for g in arches] @@ -447,6 +451,7 @@ class TestDataCenterLemonadeRouting: if "lemonade-sdk" in url: return [_lemonade_release(tag = "b1301")] # would win if pin ignored return {"tag_name": "b9637"} + mock_fetch.side_effect = _router host = rocm_host(rocm_gfx_target = "gfx908") _tag, plans = prebuilt_mod.resolve_simple_install_release_plans( From 274cc61ecb38e06322bb9feb841ae9c911ff21a4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 15 Jun 2026 06:25:49 -0700 Subject: [PATCH 6/6] Studio: harden marker handling and tighten the data-center routing gate Follow-up hardening from review: - check_prebuilt_freshness: guard a non-dict marker and a non-string source the same way effective_published_repo does, so a corrupt on-disk UNSLOTH_PREBUILT_INFO.json cannot crash update-status polling. - Data-center lemonade gate: skip on macOS explicitly. macOS never has AMD ROCm, so a stray forwarded gfx must keep the macOS bundle path. - Tests: Windows data-center routing (windows-hip + windows asset), draft/prerelease filtering in the release scan, and a direct effective_published_repo matrix including corrupt-marker fail-safes. --- .../backend/tests/test_llama_cpp_freshness.py | 26 +++++++++++++++++ studio/backend/utils/llama_cpp_freshness.py | 5 ++-- studio/install_llama_prebuilt.py | 4 ++- tests/studio/install/test_rocm_support.py | 28 +++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/studio/backend/tests/test_llama_cpp_freshness.py b/studio/backend/tests/test_llama_cpp_freshness.py index f90c4ba0e7..fdd62f9ffa 100644 --- a/studio/backend/tests/test_llama_cpp_freshness.py +++ b/studio/backend/tests/test_llama_cpp_freshness.py @@ -362,6 +362,32 @@ def test_is_behind(installed, latest, expected): assert fr.is_behind(installed, latest) is expected +_FORK = fr.DEFAULT_PUBLISHED_REPO +_LEM = fr.LEMONADE_ROCM_REPO +_GGML = "ggml-org/llama.cpp" + + +@pytest.mark.parametrize( + "marker, expected", + [ + # consumer lemonade recorded a non-fork repo -> migrate to the fork + ({"source": "lemonade", "published_repo": _GGML}, _FORK), + ({"source": "lemonade", "published_repo": _FORK}, _FORK), + # data-center lemonade already on the lemonade repo -> keep it + ({"source": "lemonade", "published_repo": _LEM}, _LEM), + # non-lemonade markers keep their own repo + ({"source": "published", "published_repo": _FORK}, _FORK), + ({"source": "upstream", "published_repo": _GGML}, _GGML), + # defensive: corrupt markers fail safe instead of crashing the poller + (None, None), + ([1, 2], None), + ({"source": ["lemonade"], "published_repo": _GGML}, _GGML), + ], +) +def test_effective_published_repo(marker, expected): + assert fr.effective_published_repo(marker) == expected + + def test_check_prebuilt_freshness_not_behind_on_mix_latest(monkeypatch, tmp_path): # Installed the mix latest: marker base tag b9596, full release_tag with sha, # GitHub latest is that same full tag. Must not report behind (sticky bug). diff --git a/studio/backend/utils/llama_cpp_freshness.py b/studio/backend/utils/llama_cpp_freshness.py index 810e8dc492..2d6ff7068d 100644 --- a/studio/backend/utils/llama_cpp_freshness.py +++ b/studio/backend/utils/llama_cpp_freshness.py @@ -275,14 +275,15 @@ def check_prebuilt_freshness( "threshold_days": int(threshold_days), } marker = read_install_marker(binary_path) - if not marker: + if not marker or not isinstance(marker, dict): return out out["has_marker"] = True # Display prefers the normalized base ("tag"); comparison below prefers the # full "release_tag" -- deliberately opposite fallbacks. A lemonade install's # "tag" is the upstream build it was made from, but it tracks lemonade's own # release counter, so show release_tag to keep installed/latest one series. - if (marker.get("source") or "").lower() == "lemonade": + source = marker.get("source") + if isinstance(source, str) and source.lower() == "lemonade": out["installed_tag"] = marker.get("release_tag") or marker.get("tag") else: out["installed_tag"] = marker.get("tag") or marker.get("release_tag") diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 0345bf6622..bc61100053 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1717,9 +1717,11 @@ def resolve_simple_install_release_plans( repo = published_repo or DEFAULT_PUBLISHED_REPO # Data-center AMD GPUs (gfx908/gfx90a) are not in the fork's per-gfx bundles; # serve them from lemonade. Catches the fork-routed fresh install and the - # lemonade-repo update re-install alike. + # lemonade-repo update re-install alike. macOS never has AMD ROCm, so a + # stray forwarded gfx there must not divert off the macOS bundle path. if ( host.is_x86_64 + and not host.is_macos and not host.has_usable_nvidia and (repo == LEMONADE_ROCM_REPO or _lemonade_datacenter_gfx(host.rocm_gfx_target)) ): diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 8a6c64052e..1b6173c5d0 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -460,6 +460,34 @@ class TestDataCenterLemonadeRouting: assert plans[0].release_tag == "b1295" assert plans[0].llama_tag == "b9637" + @patch.object(prebuilt_mod, "fetch_json", side_effect = _fetch_router) + def test_datacenter_windows_routes_to_windows_hip(self, _mock): + # Windows data-center host: lemonade "windows" asset, windows-hip kind. + host = rocm_host( + system = "Windows", + machine = "AMD64", + is_windows = True, + is_linux = False, + rocm_gfx_target = "gfx908", + ) + _tag, plans = prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, "" + ) + choice = plans[0].attempts[0] + assert choice.install_kind == "windows-hip" + assert choice.name == "llama-b1300-windows-rocm-gfx908-x64.zip" + + @patch.object(prebuilt_mod, "fetch_json") + def test_draft_and_prerelease_releases_skipped(self, mock_fetch): + # The newest list entry is a draft + prerelease; fall back to the real one. + newest = _lemonade_release(tag = "b1310", published_at = "2026-03-01T00:00:00Z") + newest["draft"], newest["prerelease"] = True, True + stable = _lemonade_release(tag = "b1300", published_at = "2026-01-15T00:00:00Z") + mock_fetch.return_value = [newest, stable] + host = rocm_host(rocm_gfx_target = "gfx908") + choice = prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") + assert choice is not None and choice.tag == "b1300" + # TEST: install_llama_prebuilt.py -- runtime_patterns_for_choice