Studio: route AMD ROCm llama.cpp installs and updates by GPU class
The fork (unslothai/llama.cpp) ships per-gfx ROCm bundles for the consumer and APU arches (gfx103X/110X/120X/1150/1151) but not the CDNA data-center cards (MI100 gfx908, MI200 gfx90a), which only lemonade publishes. - Data-center GPUs: detect gfx908/gfx90a and serve them from lemonade at install time, recording the lemonade repo in the marker so updates re-check and re-install from lemonade. Lemonade assets stay hash-exempt (functional validation only); UNSLOTH_DISABLE_LEMONADE_ROCM opts out. - Consumer GPUs: a legacy lemonade install (pre-#6225) now updates from the fork, whose per-gfx bundles match what lemonade served. effective_published_repo redirects a source=lemonade marker to the fork unless it already recorded the lemonade repo (the data-center case), so the two paths do not collide. No change for NVIDIA, non-AMD, macOS, CPU, or consumer fork installs.
This commit is contained in:
Daniel Han
parent
4176448fb8
commit
cf43072a09
5 changed files with 291 additions and 4 deletions
|
|
@ -83,6 +83,7 @@ def _write_install(
|
|||
repo: str = "unslothai/llama.cpp",
|
||||
asset: str | None = None,
|
||||
release_tag: str | None = None,
|
||||
source: str | None = None,
|
||||
) -> str:
|
||||
"""Create a fake prebuilt install tree and return the llama-server path.
|
||||
|
||||
|
|
@ -104,6 +105,8 @@ def _write_install(
|
|||
}
|
||||
if asset is not None:
|
||||
marker["asset"] = asset
|
||||
if source is not None:
|
||||
marker["source"] = source
|
||||
(dir_ / MARKER).write_text(json.dumps(marker))
|
||||
return str(binary)
|
||||
|
||||
|
|
@ -314,6 +317,98 @@ def test_status_up_to_date(monkeypatch, tmp_path):
|
|||
assert st["update_available"] is False
|
||||
|
||||
|
||||
def test_status_lemonade_marker_compares_against_fork(monkeypatch, tmp_path):
|
||||
# A pre-#6225 lemonade install whose marker recorded ggml-org (Adrenalin-only
|
||||
# Windows) must compare against the fork's latest, not ggml-org's, so it
|
||||
# migrates onto the fork's per-gfx bundle.
|
||||
binary = _write_install(
|
||||
tmp_path,
|
||||
"b9518",
|
||||
repo = "ggml-org/llama.cpp",
|
||||
asset = "llama-b1292-windows-rocm-gfx1151-x64.zip",
|
||||
source = "lemonade",
|
||||
)
|
||||
monkeypatch.setattr(upd, "_find_binary", lambda: binary)
|
||||
monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50")
|
||||
st = upd.get_update_status(force_refresh = True)
|
||||
assert st["update_available"] is True
|
||||
assert st["published_repo"] == "unslothai/llama.cpp" # not the recorded ggml-org
|
||||
assert st["latest_tag"] == "b9632-mix-2d6bd50"
|
||||
|
||||
|
||||
def test_start_update_lemonade_marker_targets_fork(monkeypatch, tmp_path):
|
||||
# start_update on the same marker re-installs from the fork with the gfx
|
||||
# recovered from the lemonade asset name.
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
binary = _write_install(
|
||||
install_dir,
|
||||
"b9518",
|
||||
repo = "ggml-org/llama.cpp",
|
||||
asset = "llama-b1292-windows-rocm-gfx1151-x64.zip",
|
||||
source = "lemonade",
|
||||
)
|
||||
monkeypatch.setattr(upd, "_find_binary", lambda: binary)
|
||||
monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")
|
||||
monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b9632-mix-2d6bd50")
|
||||
captured = {}
|
||||
_patch_installer_popen(monkeypatch, on_start = lambda cmd: captured.update(cmd = cmd))
|
||||
res = upd.start_update()
|
||||
assert res["started"] is True, res
|
||||
deadline = time.time() + 10
|
||||
while time.time() < deadline:
|
||||
if upd.get_update_status()["job"]["state"] in ("success", "error"):
|
||||
break
|
||||
time.sleep(0.05)
|
||||
cmd = captured["cmd"]
|
||||
assert "--published-repo" in cmd and "unslothai/llama.cpp" in cmd
|
||||
assert "ggml-org/llama.cpp" not in cmd
|
||||
assert cmd[cmd.index("--rocm-gfx") + 1] == "gfx1151"
|
||||
|
||||
|
||||
def test_status_datacenter_lemonade_marker_stays_on_lemonade(monkeypatch, tmp_path):
|
||||
# A data-center install (gfx908/gfx90a) recorded the lemonade repo; the fork
|
||||
# has no such bundle, so its update must keep checking lemonade.
|
||||
binary = _write_install(
|
||||
tmp_path,
|
||||
"b1292",
|
||||
repo = "lemonade-sdk/llamacpp-rocm",
|
||||
asset = "llama-b1292-ubuntu-rocm-gfx908-x64.zip",
|
||||
source = "lemonade",
|
||||
)
|
||||
monkeypatch.setattr(upd, "_find_binary", lambda: binary)
|
||||
monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b1300")
|
||||
st = upd.get_update_status(force_refresh = True)
|
||||
assert st["published_repo"] == "lemonade-sdk/llamacpp-rocm"
|
||||
assert st["latest_tag"] == "b1300"
|
||||
assert st["update_available"] is True
|
||||
|
||||
|
||||
def test_start_update_datacenter_marker_targets_lemonade(monkeypatch, tmp_path):
|
||||
install_dir = tmp_path / "llama.cpp"
|
||||
binary = _write_install(
|
||||
install_dir,
|
||||
"b1292",
|
||||
repo = "lemonade-sdk/llamacpp-rocm",
|
||||
asset = "llama-b1292-ubuntu-rocm-gfx908-x64.zip",
|
||||
source = "lemonade",
|
||||
)
|
||||
monkeypatch.setattr(upd, "_find_binary", lambda: binary)
|
||||
monkeypatch.setattr(upd, "_installer_script", lambda: tmp_path / "install_llama_prebuilt.py")
|
||||
monkeypatch.setattr(freshness, "_fetch_latest_release_tag", lambda repo, timeout = 5.0: "b1300")
|
||||
captured = {}
|
||||
_patch_installer_popen(monkeypatch, on_start = lambda cmd: captured.update(cmd = cmd))
|
||||
res = upd.start_update()
|
||||
assert res["started"] is True, res
|
||||
deadline = time.time() + 10
|
||||
while time.time() < deadline:
|
||||
if upd.get_update_status()["job"]["state"] in ("success", "error"):
|
||||
break
|
||||
time.sleep(0.05)
|
||||
cmd = captured["cmd"]
|
||||
assert "--published-repo" in cmd and "lemonade-sdk/llamacpp-rocm" in cmd
|
||||
assert cmd[cmd.index("--rocm-gfx") + 1] == "gfx908"
|
||||
|
||||
|
||||
def test_start_update_no_marker_no_prebuilt_refuses(monkeypatch, tmp_path):
|
||||
binary = tmp_path / "llama-server"
|
||||
binary.write_text("stub") # no marker
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@ _RELEASE_CACHE_TTL_SECONDS = 24 * 60 * 60
|
|||
|
||||
_INSTALL_MARKER_NAME = "UNSLOTH_PREBUILT_INFO.json"
|
||||
|
||||
# The fork now ships the per-gfx ROCm bundles lemonade used to (same gfx
|
||||
# coverage), so lemonade installs update from the fork. See effective_published_repo.
|
||||
DEFAULT_PUBLISHED_REPO = "unslothai/llama.cpp"
|
||||
LEMONADE_ROCM_REPO = "lemonade-sdk/llamacpp-rocm"
|
||||
|
||||
_marker_cache: dict[str, Optional[dict]] = {}
|
||||
_release_memo: dict[str, tuple[float, Optional[str]]] = {}
|
||||
|
||||
|
|
@ -72,6 +77,21 @@ def read_install_marker(binary_path: Optional[str]) -> Optional[dict]:
|
|||
return marker
|
||||
|
||||
|
||||
def effective_published_repo(marker: Optional[dict]) -> Optional[str]:
|
||||
"""Repo whose releases supersede this install. A pre-#6225 consumer lemonade
|
||||
install recorded a non-fork published_repo (the fork for tooling hosts, but
|
||||
ggml-org for Adrenalin-only Windows), yet the fork now ships those per-gfx
|
||||
ROCm bundles; route its updates to the fork. Data-center installs (gfx908/
|
||||
gfx90a) recorded the lemonade repo and keep updating from it (the fork has no
|
||||
such bundle). Non-lemonade markers keep their published_repo."""
|
||||
if not marker:
|
||||
return None
|
||||
repo = marker.get("published_repo")
|
||||
if (marker.get("source") or "").lower() == "lemonade" and repo != LEMONADE_ROCM_REPO:
|
||||
return DEFAULT_PUBLISHED_REPO
|
||||
return repo
|
||||
|
||||
|
||||
def _cache_path_for(repo: str) -> Path:
|
||||
safe = repo.replace("/", "__")
|
||||
return _cache_dir() / f"{safe}.json"
|
||||
|
|
@ -260,7 +280,8 @@ def check_prebuilt_freshness(
|
|||
# full "release_tag" -- deliberately opposite fallbacks.
|
||||
out["installed_tag"] = marker.get("tag") or marker.get("release_tag")
|
||||
out["installed_at_utc"] = marker.get("installed_at_utc")
|
||||
out["published_repo"] = marker.get("published_repo")
|
||||
# Lemonade installs compare against (and later re-install from) the fork.
|
||||
out["published_repo"] = effective_published_repo(marker)
|
||||
|
||||
# The marker records both a normalized base tag ("tag", e.g. b9596) and the
|
||||
# full release tag ("release_tag", e.g. b9596-mix-<sha>). Compare against the
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import structlog
|
|||
from utils.llama_cpp_freshness import (
|
||||
_INSTALL_MARKER_NAME,
|
||||
check_prebuilt_freshness,
|
||||
effective_published_repo,
|
||||
latest_published_release,
|
||||
parse_base_build,
|
||||
read_install_marker,
|
||||
|
|
@ -294,7 +295,7 @@ def get_update_status(*, force_refresh: bool = False) -> dict:
|
|||
if src is not None:
|
||||
return src
|
||||
|
||||
repo = (marker or {}).get("published_repo") or DEFAULT_PUBLISHED_REPO
|
||||
repo = effective_published_repo(marker) or DEFAULT_PUBLISHED_REPO
|
||||
|
||||
if force_refresh and repo:
|
||||
# Prime the cache so the freshness read below sees the newest tag.
|
||||
|
|
@ -505,7 +506,7 @@ def start_update() -> dict:
|
|||
"job": status["job"],
|
||||
}
|
||||
install_dir = _install_dir_for(binary)
|
||||
repo = marker.get("published_repo") or DEFAULT_PUBLISHED_REPO
|
||||
repo = effective_published_repo(marker) or DEFAULT_PUBLISHED_REPO
|
||||
from_tag = marker.get("tag") or marker.get("release_tag")
|
||||
asset = marker.get("asset")
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -139,6 +139,11 @@ DEFAULT_PUBLISHED_SHA256_ASSET = os.environ.get(
|
|||
UPSTREAM_REPO = "ggml-org/llama.cpp"
|
||||
UPSTREAM_RELEASES_API = f"https://api.github.com/repos/{UPSTREAM_REPO}/releases/latest"
|
||||
|
||||
# CDNA data-center arches lemonade ships ROCm bundles for that the fork's per-gfx
|
||||
# bundles do not (MI100 / MI200). Consumer/APU arches are served by the fork.
|
||||
LEMONADE_ROCM_REPO = "lemonade-sdk/llamacpp-rocm"
|
||||
_LEMONADE_DATACENTER_ARCHES = ("gfx908", "gfx90a")
|
||||
|
||||
|
||||
TEST_MODEL_URL = "https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K.gguf"
|
||||
TEST_MODEL_SHA256 = "270cba1bd5109f42d03350f60406024560464db173c0e387d91f0426d3bd256d"
|
||||
|
|
@ -1588,6 +1593,89 @@ def pinned_macos_release_tag(host: HostInfo, repo: str) -> str | None:
|
|||
return _PINNED_MACOS_FALLBACK_TAG
|
||||
|
||||
|
||||
def _lemonade_datacenter_gfx(gfx_target: str | None) -> str | None:
|
||||
"""The host gfx if it is a data-center arch only lemonade covers, else None."""
|
||||
gfx = (gfx_target or "").lower().strip()
|
||||
return gfx if gfx in _LEMONADE_DATACENTER_ARCHES else None
|
||||
|
||||
|
||||
def _lemonade_release_api_for(llama_tag: str) -> str:
|
||||
"""Lemonade release API URL. Lemonade uses its own tag series, so a pinned
|
||||
fork/ggml-org tag will 404 here; only 'latest' or a lemonade tag resolve."""
|
||||
tag = (llama_tag or "").strip()
|
||||
if not tag or tag.lower() == "latest":
|
||||
return f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/latest"
|
||||
return (
|
||||
f"https://api.github.com/repos/{LEMONADE_ROCM_REPO}/releases/tags/"
|
||||
f"{urllib.parse.quote(tag, safe = '')}"
|
||||
)
|
||||
|
||||
|
||||
def resolve_lemonade_rocm_choice(
|
||||
host: HostInfo, os_prefix: str, install_kind: str, llama_tag: str = "latest"
|
||||
) -> AssetChoice | None:
|
||||
"""Lemonade ROCm bundle for a data-center AMD GPU (gfx908/gfx90a) the fork
|
||||
does not ship. None for any other arch or on a fetch/asset/host miss.
|
||||
Lemonade assets are not in the approved-hash manifest, so integrity is
|
||||
functional validation only; UNSLOTH_DISABLE_LEMONADE_ROCM opts out."""
|
||||
gfx = _lemonade_datacenter_gfx(host.rocm_gfx_target)
|
||||
if gfx is None:
|
||||
return None
|
||||
if os.environ.get("UNSLOTH_DISABLE_LEMONADE_ROCM", "").strip().lower() in ("1", "true", "yes"):
|
||||
log("UNSLOTH_DISABLE_LEMONADE_ROCM is set; skipping lemonade-sdk prebuilt")
|
||||
return None
|
||||
try:
|
||||
release = fetch_json(_lemonade_release_api_for(llama_tag))
|
||||
except Exception as exc:
|
||||
log(f"Could not fetch {LEMONADE_ROCM_REPO} release ({exc}); skipping lemonade prebuilt")
|
||||
return None
|
||||
release_tag = release.get("tag_name") if isinstance(release, dict) else None
|
||||
if not isinstance(release_tag, str) or not release_tag:
|
||||
return None
|
||||
asset_name = f"llama-{release_tag}-{os_prefix}-rocm-{gfx}-x64.zip"
|
||||
asset_url = release_asset_map(release).get(asset_name)
|
||||
# browser_download_url for a release asset is always github.com/<repo>/download.
|
||||
trusted = (asset_url or "").startswith(
|
||||
f"https://github.com/{LEMONADE_ROCM_REPO}/releases/download/"
|
||||
)
|
||||
if not asset_url or not trusted:
|
||||
log(f"{LEMONADE_ROCM_REPO}@{release_tag} missing or untrusted {asset_name!r}; skipping")
|
||||
return None
|
||||
return AssetChoice(
|
||||
repo = LEMONADE_ROCM_REPO,
|
||||
tag = release_tag,
|
||||
name = asset_name,
|
||||
url = asset_url,
|
||||
source_label = "lemonade",
|
||||
install_kind = install_kind,
|
||||
selection_log = [f"rocm_selection: data-center GPU {host.rocm_gfx_target} -> {asset_name}"],
|
||||
)
|
||||
|
||||
|
||||
def _lemonade_release_plans(
|
||||
llama_tag: str, host: HostInfo
|
||||
) -> tuple[str, list[InstallReleasePlan]]:
|
||||
"""Single-attempt plan for a data-center AMD GPU, sourced from lemonade. The
|
||||
marker records the lemonade repo/tag so updates re-check and re-install it."""
|
||||
requested_tag = normalized_requested_llama_tag(llama_tag)
|
||||
os_prefix, install_kind = (
|
||||
("windows", "windows-hip") if host.is_windows else ("ubuntu", "linux-rocm")
|
||||
)
|
||||
choice = resolve_lemonade_rocm_choice(host, os_prefix, install_kind, llama_tag = requested_tag)
|
||||
if choice is None:
|
||||
raise PrebuiltFallback("no lemonade ROCm prebuilt for this data-center GPU")
|
||||
plan = InstallReleasePlan(
|
||||
requested_tag = requested_tag,
|
||||
llama_tag = choice.tag,
|
||||
release_tag = choice.tag,
|
||||
attempts = [choice],
|
||||
approved_checksums = synthetic_checksums_for_release(
|
||||
LEMONADE_ROCM_REPO, choice.tag, choice.tag
|
||||
),
|
||||
)
|
||||
return requested_tag, [plan]
|
||||
|
||||
|
||||
def resolve_simple_install_release_plans(
|
||||
llama_tag: str,
|
||||
host: HostInfo,
|
||||
|
|
@ -1597,6 +1685,15 @@ def resolve_simple_install_release_plans(
|
|||
max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS,
|
||||
) -> tuple[str, list[InstallReleasePlan]]:
|
||||
repo = published_repo or DEFAULT_PUBLISHED_REPO
|
||||
# Data-center AMD GPUs (gfx908/gfx90a) are not in the fork's per-gfx bundles;
|
||||
# serve them from lemonade. Catches the fork-routed fresh install and the
|
||||
# lemonade-repo update re-install alike.
|
||||
if (
|
||||
host.is_x86_64
|
||||
and not host.has_usable_nvidia
|
||||
and (repo == LEMONADE_ROCM_REPO or _lemonade_datacenter_gfx(host.rocm_gfx_target))
|
||||
):
|
||||
return _lemonade_release_plans(llama_tag, host)
|
||||
# The fork (unslothai) ships a manifest describing every bundle's GPU/arch
|
||||
# coverage, so all fork hosts select from it. Upstream (ggml-org) ships no
|
||||
# manifest and is selected by asset filename in the loop below.
|
||||
|
|
@ -6857,7 +6954,9 @@ def main() -> int:
|
|||
else:
|
||||
payload = {
|
||||
"prebuilt_available": True,
|
||||
"repo": repo,
|
||||
# The plan's repo, not the host default: a data-center GPU
|
||||
# routed to the fork is actually served from lemonade.
|
||||
"repo": plans[0].approved_checksums.repo or repo,
|
||||
"release_tag": plans[0].release_tag,
|
||||
"llama_tag": plans[0].llama_tag,
|
||||
"asset": choice.name,
|
||||
|
|
|
|||
|
|
@ -330,6 +330,77 @@ class TestResolveUpstreamAssetChoice:
|
|||
resolve_upstream_asset_choice(host, LLAMA_TAG)
|
||||
|
||||
|
||||
# TEST: install_llama_prebuilt.py -- data-center ROCm (lemonade) routing
|
||||
|
||||
|
||||
def _lemonade_release(tag = "b1300"):
|
||||
base = f"https://github.com/lemonade-sdk/llamacpp-rocm/releases/download/{tag}"
|
||||
names = [
|
||||
f"llama-{tag}-ubuntu-rocm-gfx908-x64.zip",
|
||||
f"llama-{tag}-ubuntu-rocm-gfx90a-x64.zip",
|
||||
f"llama-{tag}-windows-rocm-gfx908-x64.zip",
|
||||
]
|
||||
return {
|
||||
"tag_name": tag,
|
||||
"assets": [{"name": n, "browser_download_url": f"{base}/{n}"} for n in names],
|
||||
}
|
||||
|
||||
|
||||
class TestDataCenterLemonadeRouting:
|
||||
"""gfx908/gfx90a (MI100/MI200) are not in the fork's per-gfx bundles, so they
|
||||
are served from lemonade at install and update time. Consumer arches and
|
||||
NVIDIA hosts must not be routed here."""
|
||||
|
||||
@patch.object(prebuilt_mod, "fetch_json", return_value = _lemonade_release())
|
||||
def test_datacenter_linux_routes_to_lemonade(self, _mock):
|
||||
host = rocm_host(rocm_gfx_target = "gfx908")
|
||||
_tag, plans = prebuilt_mod.resolve_simple_install_release_plans(
|
||||
"latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, ""
|
||||
)
|
||||
choice = plans[0].attempts[0]
|
||||
assert choice.source_label == "lemonade"
|
||||
assert choice.repo == prebuilt_mod.LEMONADE_ROCM_REPO
|
||||
assert choice.install_kind == "linux-rocm" and "gfx908" in choice.name
|
||||
# Marker repo (published_repo) is lemonade, so updates re-check lemonade.
|
||||
assert plans[0].approved_checksums.repo == prebuilt_mod.LEMONADE_ROCM_REPO
|
||||
|
||||
@patch.object(prebuilt_mod, "fetch_json", return_value = _lemonade_release())
|
||||
def test_update_via_lemonade_repo_routes_to_lemonade(self, _mock):
|
||||
# The update path re-invokes with --published-repo lemonade + --rocm-gfx.
|
||||
host = rocm_host(rocm_gfx_target = "gfx90a")
|
||||
_tag, plans = prebuilt_mod.resolve_simple_install_release_plans(
|
||||
"latest", host, prebuilt_mod.LEMONADE_ROCM_REPO, ""
|
||||
)
|
||||
assert plans[0].attempts[0].name.endswith("ubuntu-rocm-gfx90a-x64.zip")
|
||||
|
||||
def test_consumer_arch_not_routed_to_lemonade(self, monkeypatch):
|
||||
# gfx1151 is a fork family: lemonade returns None and the dispatch must
|
||||
# fall through to the fork manifest path, not lemonade.
|
||||
host = rocm_host(rocm_gfx_target = "gfx1151")
|
||||
assert prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") is None
|
||||
sentinel = ("forkpath", [])
|
||||
monkeypatch.setattr(prebuilt_mod, "_fork_manifest_release_plans", lambda *a, **k: sentinel)
|
||||
assert prebuilt_mod.resolve_simple_install_release_plans(
|
||||
"latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, ""
|
||||
) == sentinel
|
||||
|
||||
def test_nvidia_with_stray_gfx_not_routed_to_lemonade(self, monkeypatch):
|
||||
host = nvidia_host(has_rocm = True, rocm_gfx_target = "gfx908")
|
||||
sentinel = ("forkpath", [])
|
||||
monkeypatch.setattr(prebuilt_mod, "_fork_manifest_release_plans", lambda *a, **k: sentinel)
|
||||
assert prebuilt_mod.resolve_simple_install_release_plans(
|
||||
"latest", host, prebuilt_mod.DEFAULT_PUBLISHED_REPO, ""
|
||||
) == sentinel
|
||||
|
||||
@patch.object(prebuilt_mod, "fetch_json")
|
||||
def test_untrusted_asset_url_skipped(self, mock_fetch):
|
||||
rel = _lemonade_release()
|
||||
rel["assets"][0]["browser_download_url"] = "https://evil.example.com/x.zip"
|
||||
mock_fetch.return_value = rel
|
||||
host = rocm_host(rocm_gfx_target = "gfx908")
|
||||
assert prebuilt_mod.resolve_lemonade_rocm_choice(host, "ubuntu", "linux-rocm") is None
|
||||
|
||||
|
||||
# TEST: install_llama_prebuilt.py -- runtime_patterns_for_choice
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue