From 10d7ac5eb20b728bb8fa3cf58e6156f3c2ce0ce3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 15 Jun 2026 06:00:40 -0700 Subject: [PATCH] Studio: align data-center lemonade selection with the time-ordered resolver Match the lemonade ROCm path to how the fork/upstream resolver and the update banner already pick releases, so install and update agree: - Scan recent lemonade releases newest-by-publish-time instead of calling /releases/latest, which sorts by commit date and can lag. The banner's latest_published_release() already does this, so they no longer disagree and leave MI100/MI200 hosts with a sticky banner or a downgrade. - Walk back past a newest release that is missing this GPU's asset to an older complete one, mirroring the fork resolver's release fallback, instead of dropping straight to a HIP source build. - Honor a pinned --published-release-tag for gfx908/gfx90a so the pin/rollback path works the same as the fork/upstream branches. - effective_published_repo: guard against a non-dict marker or non-string source so corrupt on-disk markers cannot crash update-status polling. Tests cover walk-back, publish-time ordering, and pin selection. --- studio/backend/utils/llama_cpp_freshness.py | 6 +- studio/install_llama_prebuilt.py | 105 ++++++++++++-------- tests/studio/install/test_rocm_support.py | 59 +++++++++-- 3 files changed, 116 insertions(+), 54 deletions(-) diff --git a/studio/backend/utils/llama_cpp_freshness.py b/studio/backend/utils/llama_cpp_freshness.py index a5ab59c7dd..810e8dc492 100644 --- a/studio/backend/utils/llama_cpp_freshness.py +++ b/studio/backend/utils/llama_cpp_freshness.py @@ -84,10 +84,12 @@ def effective_published_repo(marker: Optional[dict]) -> Optional[str]: ROCm bundles; route its updates to the fork. Data-center installs (gfx908/ gfx90a) recorded the lemonade repo and keep updating from it (the fork has no such bundle). Non-lemonade markers keep their published_repo.""" - if not marker: + if not isinstance(marker, dict): return None repo = marker.get("published_repo") - if (marker.get("source") or "").lower() == "lemonade" and repo != LEMONADE_ROCM_REPO: + source = marker.get("source") + source = source.lower() if isinstance(source, str) else "" + if source == "lemonade" and repo != LEMONADE_ROCM_REPO: return DEFAULT_PUBLISHED_REPO return repo diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index dc2de31773..ee86a985b1 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1599,16 +1599,34 @@ def _lemonade_datacenter_gfx(gfx_target: str | None) -> str | None: return gfx if gfx in _LEMONADE_DATACENTER_ARCHES else None -def _lemonade_release_api_for(llama_tag: str) -> str: - """Lemonade release API URL. Lemonade uses its own tag series, so a pinned - fork/ggml-org tag will 404 here; only 'latest' or a lemonade tag resolve.""" +def _lemonade_releases_newest_first(llama_tag: str) -> list[dict[str, Any]]: + """Lemonade releases to try, newest-published first. A pinned tag yields just + that release; 'latest' scans the recent list ordered by publish time -- the + same ordering the update banner uses, so detection and install agree (the + /releases/latest pointer sorts by commit date and can lag). Lemonade uses its + own tag series, so a pinned fork/ggml-org tag simply 404s to an empty list.""" tag = (llama_tag or "").strip() - if not tag or tag.lower() == "latest": - return f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/latest" - return ( - f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/tags/" - f"{urllib.parse.quote(tag, safe = '')}" - ) + api = f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases" + try: + if tag and tag.lower() != "latest": + return [fetch_json(f"{api}/tags/{urllib.parse.quote(tag, safe = '')}")] + payload = fetch_json(f"{api}?per_page=30") + except Exception as exc: + log(f"Could not fetch {LEMONADE_ROCM_REPO} releases ({exc}); skipping lemonade prebuilt") + return [] + if not isinstance(payload, list): + return [] + rels = [ + r + for r in payload + if isinstance(r, dict) + and not r.get("draft") + and not r.get("prerelease") + and isinstance(r.get("tag_name"), str) + and r.get("tag_name") + ] + rels.sort(key = lambda r: r.get("published_at") or "", reverse = True) + return rels def resolve_lemonade_rocm_choice( @@ -1618,53 +1636,56 @@ def resolve_lemonade_rocm_choice( llama_tag: str = "latest", ) -> AssetChoice | None: """Lemonade ROCm bundle for a data-center AMD GPU (gfx908/gfx90a) the fork - does not ship. None for any other arch or on a fetch/asset/host miss. - Lemonade assets are not in the approved-hash manifest, so integrity is - functional validation only; UNSLOTH_DISABLE_LEMONADE_ROCM opts out.""" + does not ship. Walks recent lemonade releases newest-first and takes the + first carrying this GPU's asset, so a partial newest nightly falls back to an + older complete one (mirrors the fork resolver). None for any other arch, an + opt-out, or when no scanned release has the asset. Lemonade assets are not in + the approved-hash manifest, so integrity is functional validation only.""" gfx = _lemonade_datacenter_gfx(host.rocm_gfx_target) if gfx is None: return None if os.environ.get("UNSLOTH_DISABLE_LEMONADE_ROCM", "").strip().lower() in ("1", "true", "yes"): log("UNSLOTH_DISABLE_LEMONADE_ROCM is set; skipping lemonade-sdk prebuilt") return None - try: - release = fetch_json(_lemonade_release_api_for(llama_tag)) - except Exception as exc: - log(f"Could not fetch {LEMONADE_ROCM_REPO} release ({exc}); skipping lemonade prebuilt") - return None - release_tag = release.get("tag_name") if isinstance(release, dict) else None - if not isinstance(release_tag, str) or not release_tag: - return None - asset_name = f"llama-{release_tag}-{os_prefix}-rocm-{gfx}-x64.zip" - asset_url = release_asset_map(release).get(asset_name) - # browser_download_url for a release asset is always github.com//download. - trusted = (asset_url or "").startswith( - f"https://github.com/{LEMONADE_ROCM_REPO}/releases/download/" - ) - if not asset_url or not trusted: - log(f"{LEMONADE_ROCM_REPO}@{release_tag} missing or untrusted {asset_name!r}; skipping") - return None - return AssetChoice( - repo = LEMONADE_ROCM_REPO, - tag = release_tag, - name = asset_name, - url = asset_url, - source_label = "lemonade", - install_kind = install_kind, - selection_log = [f"rocm_selection: data-center GPU {host.rocm_gfx_target} -> {asset_name}"], - ) + for release in _lemonade_releases_newest_first(llama_tag): + release_tag = release.get("tag_name") if isinstance(release, dict) else None + if not isinstance(release_tag, str) or not release_tag: + continue + asset_name = f"llama-{release_tag}-{os_prefix}-rocm-{gfx}-x64.zip" + asset_url = release_asset_map(release).get(asset_name) + # browser_download_url for a release asset is always github.com//download. + if asset_url and asset_url.startswith( + f"https://github.com/{LEMONADE_ROCM_REPO}/releases/download/" + ): + return AssetChoice( + repo = LEMONADE_ROCM_REPO, + tag = release_tag, + name = asset_name, + url = asset_url, + source_label = "lemonade", + install_kind = install_kind, + selection_log = [ + f"rocm_selection: data-center GPU {host.rocm_gfx_target} -> {asset_name}" + ], + ) + log(f"no recent {LEMONADE_ROCM_REPO} release carries a {gfx} asset; skipping lemonade prebuilt") + return None -def _lemonade_release_plans(llama_tag: str, host: HostInfo) -> tuple[str, list[InstallReleasePlan]]: +def _lemonade_release_plans( + llama_tag: str, host: HostInfo, published_release_tag: str = "" +) -> tuple[str, list[InstallReleasePlan]]: """Single-attempt plan for a data-center AMD GPU, sourced from lemonade. release_tag is lemonade's own counter so updates compare against lemonade; llama_tag is a real upstream tag because the source tree is hydrated from - ggml-org by it (lemonade's counter is not a ggml-org ref).""" + ggml-org by it (lemonade's counter is not a ggml-org ref). A pinned + published_release_tag selects a specific lemonade release for rollback.""" requested_tag = normalized_requested_llama_tag(llama_tag) + lemonade_tag = published_release_tag.strip() or requested_tag os_prefix, install_kind = ( ("windows", "windows-hip") if host.is_windows else ("ubuntu", "linux-rocm") ) - choice = resolve_lemonade_rocm_choice(host, os_prefix, install_kind, llama_tag = requested_tag) + choice = resolve_lemonade_rocm_choice(host, os_prefix, install_kind, llama_tag = lemonade_tag) if choice is None: raise PrebuiltFallback("no lemonade ROCm prebuilt for this data-center GPU") try: @@ -1700,7 +1721,7 @@ def resolve_simple_install_release_plans( and not host.has_usable_nvidia and (repo == LEMONADE_ROCM_REPO or _lemonade_datacenter_gfx(host.rocm_gfx_target)) ): - return _lemonade_release_plans(llama_tag, host) + return _lemonade_release_plans(llama_tag, host, published_release_tag) # The fork (unslothai) ships a manifest describing every bundle's GPU/arch # coverage, so all fork hosts select from it. Upstream (ggml-org) ships no # manifest and is selected by asset filename in the loop below. diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 66d8261941..338cbc1cca 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -333,23 +333,22 @@ class TestResolveUpstreamAssetChoice: # TEST: install_llama_prebuilt.py -- data-center ROCm (lemonade) routing -def _lemonade_release(tag = "b1300"): +def _lemonade_release(tag = "b1300", published_at = "2026-01-01T00:00:00Z", gfxs = None): base = f"https://github.com/lemonade-sdk/llamacpp-rocm/releases/download/{tag}" - names = [ - f"llama-{tag}-ubuntu-rocm-gfx908-x64.zip", - f"llama-{tag}-ubuntu-rocm-gfx90a-x64.zip", - f"llama-{tag}-windows-rocm-gfx908-x64.zip", - ] + arches = gfxs if gfxs is not None else ["gfx908", "gfx90a"] + names = [f"llama-{tag}-ubuntu-rocm-{g}-x64.zip" for g in arches] + names += [f"llama-{tag}-windows-rocm-{g}-x64.zip" for g in arches] return { "tag_name": tag, + "published_at": published_at, "assets": [{"name": n, "browser_download_url": f"{base}/{n}"} for n in names], } def _fetch_router(url): - # Lemonade lookup vs the upstream-latest lookup the plan does for source. + # Lemonade list lookup ('latest') vs upstream-latest lookup for source. if "lemonade-sdk" in url: - return _lemonade_release() + return [_lemonade_release()] if "?per_page" in url else _lemonade_release() return {"tag_name": "b9637"} # ggml-org upstream latest (for source hydration) @@ -410,12 +409,52 @@ class TestDataCenterLemonadeRouting: @patch.object(prebuilt_mod, "fetch_json") def test_untrusted_asset_url_skipped(self, mock_fetch): - rel = _lemonade_release() + rel = _lemonade_release(gfxs = ["gfx908"]) rel["assets"][0]["browser_download_url"] = "https://evil.example.com/x.zip" - mock_fetch.return_value = rel + mock_fetch.return_value = [rel] host = rocm_host(rocm_gfx_target = "gfx908") assert prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") is None + @patch.object(prebuilt_mod, "fetch_json") + def test_walk_back_past_partial_newest(self, mock_fetch): + # Newest release has no gfx908 asset yet -> fall back to the older one + # that does, mirroring the fork resolver's release walk-back. + newest = _lemonade_release(tag = "b1305", published_at = "2026-02-01T00:00:00Z", gfxs = []) + older = _lemonade_release(tag = "b1300", published_at = "2026-01-15T00:00:00Z") + mock_fetch.return_value = [newest, older] + host = rocm_host(rocm_gfx_target = "gfx908") + choice = prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") + assert choice is not None and choice.tag == "b1300" + + @patch.object(prebuilt_mod, "fetch_json") + def test_newest_by_publish_time_wins(self, mock_fetch): + # List order is not release order; pick the newest published_at. + mock_fetch.return_value = [ + _lemonade_release(tag = "b1290", published_at = "2026-01-01T00:00:00Z"), + _lemonade_release(tag = "b1301", published_at = "2026-03-01T00:00:00Z"), + ] + host = rocm_host(rocm_gfx_target = "gfx908") + choice = prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") + assert choice is not None and choice.tag == "b1301" + + @patch.object(prebuilt_mod, "fetch_json") + def test_pinned_release_tag_selected(self, mock_fetch): + # A pinned lemonade release_tag resolves that exact release (tags endpoint), + # while source still hydrates from a real upstream tag. + def _router(url): + if "lemonade-sdk" in url and "/tags/b1295" in url: + return _lemonade_release(tag = "b1295") + if "lemonade-sdk" in url: + return [_lemonade_release(tag = "b1301")] # would win if pin ignored + return {"tag_name": "b9637"} + mock_fetch.side_effect = _router + host = rocm_host(rocm_gfx_target = "gfx908") + _tag, plans = prebuilt_mod.resolve_simple_install_release_plans( + "latest", host, prebuilt_mod.LEMONADE_ROCM_REPO, "b1295" + ) + assert plans[0].release_tag == "b1295" + assert plans[0].llama_tag == "b9637" + # TEST: install_llama_prebuilt.py -- runtime_patterns_for_choice