diff --git a/studio/backend/tests/test_lemonade_llamacpp_rocm_bins_mock.py b/studio/backend/tests/test_lemonade_llamacpp_rocm_bins_mock.py index 3443f53a12..57bbc84fb0 100644 --- a/studio/backend/tests/test_lemonade_llamacpp_rocm_bins_mock.py +++ b/studio/backend/tests/test_lemonade_llamacpp_rocm_bins_mock.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Validates that the installer resolves lemonade ROCm prebuilt assets. +"""Validates that the installer correctly resolves lemonade ROCm prebuilt assets. -Uses a faked HostInfo so no AMD GPU is needed. The lemonade GitHub API calls -are stubbed so the suite runs offline and isn't subject to rate limits. +Uses a faked HostInfo so no AMD GPU is needed. Network calls to the lemonade +GitHub API are stubbed out so the suite runs without internet access and is +not subject to rate limits. """ from __future__ import annotations @@ -32,7 +33,7 @@ if resolve_lemonade_rocm_choice is None or _LEMONADE_GFX_FAMILIES is None: @pytest.fixture(autouse = True) def _clear_lemonade_release_cache(): """Prevent cross-test pollution of the lemonade release lru_cache when - tests vary the fetch_json mock return value.""" + future tests vary the fetch_json mock return value.""" _cache = getattr(_mod, "_fetch_lemonade_release_cached", None) if _cache is not None and hasattr(_cache, "cache_clear"): _cache.cache_clear() @@ -89,7 +90,9 @@ def _lookup_family(gfx: str) -> str | None: return None +# --------------------------------------------------------------------------- # GPU family mapping +# --------------------------------------------------------------------------- @pytest.mark.parametrize( @@ -111,7 +114,9 @@ def test_unknown_gpu_not_in_families(): assert _lookup_family("gfx999") is None +# --------------------------------------------------------------------------- # Asset resolution - hits real lemonade GitHub API +# --------------------------------------------------------------------------- @pytest.mark.parametrize( @@ -141,59 +146,71 @@ def test_unknown_gpu_falls_through_to_upstream(): assert result is None -# Simple-policy dispatcher must plan a lemonade ROCm attempt for AMD-only hosts. -# This is the path setup.sh invokes (via --simple-policy), so the lemonade -# integration is useless if it isn't wired in here. +# --------------------------------------------------------------------------- +# The Linux attempt builder must plan a lemonade ROCm attempt for AMD-only hosts. +# This is the path setup.sh actually invokes (fork hosts now select from the +# manifest), so the lemonade integration is useless if it isn't wired in here. +# --------------------------------------------------------------------------- -direct_linux_release_plan = getattr(_mod, "direct_linux_release_plan", None) +_linux_published_attempts = getattr(_mod, "_linux_published_attempts", None) direct_upstream_release_plan = getattr(_mod, "direct_upstream_release_plan", None) +PublishedLlamaArtifact = _mod.PublishedLlamaArtifact +PublishedReleaseBundle = _mod.PublishedReleaseBundle -def _stub_unsloth_release(release_tag: str = "b9022") -> dict: - # Minimal payload parse_direct_linux_release_bundle accepts. It needs at - # least one `app-{label}-linux-x64*.tar.gz` asset to recognise the bundle; - # we ship a bare CPU one so the planner has a baseline non-ROCm fallback. - asset_name = f"app-{release_tag}-linux-x64.tar.gz" - return { - "tag_name": release_tag, - "name": release_tag, - "assets": [ - { - "name": asset_name, - "browser_download_url": f"https://example.invalid/{asset_name}", - }, - ], - } + +def _rocm_bundle(gfx_family: str, mapped_targets: list[str]) -> "PublishedReleaseBundle": + """A fork manifest bundle exposing a per-gfx linux-rocm artifact, so + published_rocm_choice_for_host can match the host before the lemonade + fallback is appended.""" + asset_name = f"app-b9457-linux-x64-rocm-{gfx_family}.tar.gz" + artifact = PublishedLlamaArtifact( + asset_name = asset_name, + install_kind = "linux-rocm", + runtime_line = None, + coverage_class = None, + supported_sms = [], + min_sm = None, + max_sm = None, + bundle_profile = None, + rank = 1000, + gfx_target = gfx_family, + mapped_targets = mapped_targets, + ) + return PublishedReleaseBundle( + repo = "unslothai/llama.cpp", + release_tag = "v1.0", + upstream_tag = "b9457", + assets = {asset_name: f"https://example.invalid/{asset_name}"}, + artifacts = [artifact], + ) @pytest.mark.skipif( - direct_linux_release_plan is None, - reason = "simple-policy dispatcher not present on this branch", + _linux_published_attempts is None, + reason = "Linux attempt builder not present on this branch", ) -def test_simple_policy_plans_lemonade_for_rocm_host(): +def test_linux_attempts_include_fork_rocm_and_lemonade_for_rocm_host(): host = _make_rocm_host("gfx1151") + bundle = _rocm_bundle("gfx1151", ["gfx1151"]) with patch.object(_mod, "fetch_json", return_value = _stub_lemonade_release()): - plan = direct_linux_release_plan( - _stub_unsloth_release(), - host, - "unslothai/llama.cpp", - "latest", - ) - assert plan is not None, "ROCm host should not be skipped by simple-policy planner" - kinds = [a.install_kind for a in plan.attempts] - assert ( - "linux-rocm" in kinds - ), f"simple-policy planner did not include a lemonade ROCm attempt; got {kinds}" - rocm_attempt = next(a for a in plan.attempts if a.install_kind == "linux-rocm") - assert rocm_attempt.source_label == "lemonade" - assert "gfx1151" in rocm_attempt.name + attempts = _linux_published_attempts(host, bundle, "latest") + kinds = [a.install_kind for a in attempts] + assert "linux-rocm" in kinds, f"builder did not include any linux-rocm attempt; got {kinds}" + sources = {a.source_label for a in attempts if a.install_kind == "linux-rocm"} + # The fork's own per-gfx bundle is preferred, with the lemonade prebuilt as + # the fallback -- both must be present for a covered ROCm host. + assert "published" in sources, f"fork ROCm bundle missing; got {sources}" + assert "lemonade" in sources, f"lemonade ROCm fallback missing; got {sources}" + lemonade_attempt = next(a for a in attempts if a.source_label == "lemonade") + assert "gfx1151" in lemonade_attempt.name @pytest.mark.skipif( direct_upstream_release_plan is None, - reason = "simple-policy dispatcher not present on this branch", + reason = "direct release planners not present on this branch", ) -def test_simple_policy_plans_lemonade_for_windows_hip_host(): +def test_direct_upstream_plan_includes_lemonade_for_windows_hip_host(): host = _make_rocm_host("gfx1151", windows = True) release = { "tag_name": "b9022", @@ -204,16 +221,14 @@ def test_simple_policy_plans_lemonade_for_windows_hip_host(): plan = direct_upstream_release_plan(release, host, "ggml-org/llama.cpp", "latest") assert plan is not None, "Windows ROCm host should plan a lemonade HIP attempt" kinds = [a.install_kind for a in plan.attempts] - assert ( - "windows-hip" in kinds - ), f"simple-policy planner did not include a lemonade HIP attempt; got {kinds}" + assert "windows-hip" in kinds, f"planner did not include a lemonade HIP attempt; got {kinds}" @pytest.mark.skipif( direct_upstream_release_plan is None, - reason = "simple-policy dispatcher not present on this branch", + reason = "direct release planners not present on this branch", ) -def test_simple_policy_windows_hip_falls_back_to_upstream_when_lemonade_unavailable(): +def test_windows_hip_falls_back_to_upstream_when_lemonade_unavailable(): """If lemonade returns None (e.g. gfx999 or transient API failure), the planner must still include the upstream HIP asset rather than silently downgrading to CPU.""" host = _make_rocm_host("gfx999", windows = True) @@ -247,8 +262,9 @@ def test_lemonade_release_api_url_pinned_tag(): def test_lemonade_release_api_url_encodes_tag(): - """Slashes / hashes in the tag must be URL-encoded so the URL can't be - reshaped (defence in depth -- tags should already be sanitised upstream).""" + """Unexpected slashes / hashes in the tag must be URL-encoded so the URL + cannot be reshaped (defence in depth -- tags should already be sanitised + upstream).""" url = _mod._lemonade_release_api_for("b1260/../latest") assert "/releases/tags/b1260%2F..%2Flatest" in url assert "//latest" not in url.split("/releases/tags/", 1)[1] @@ -263,9 +279,9 @@ def test_lemonade_resolver_skipped_by_opt_out_env(monkeypatch): def test_lemonade_resolver_rejects_non_github_url(monkeypatch): - """If the GitHub API response contained an off-host download URL, the - resolver must refuse it (lemonade assets aren't in the approved-hash - manifest).""" + """If the GitHub API response somehow contained an off-host download URL, + the resolver must refuse to use it (lemonade assets are not in the + approved-hash manifest).""" bad_release = { "tag_name": _STUB_TAG, "assets": [ @@ -289,7 +305,7 @@ def test_lemonade_resolver_rejects_http_scheme(): def test_lemonade_resolver_accepts_github_cdn(): - # Real GitHub release CDN URLs carry the /github-production-release-asset- prefix + # Real GitHub release CDN URLs carry the /github-production-release-asset- prefix. assert _mod._is_trusted_github_release_url( "https://objects.githubusercontent.com/github-production-release-asset-abc123/456/789?token=x", "lemonade-sdk/llamacpp-rocm", @@ -297,7 +313,7 @@ def test_lemonade_resolver_accepts_github_cdn(): def test_lemonade_resolver_rejects_arbitrary_cdn_path(): - # A CDN URL without the release-asset path prefix must be rejected + # A CDN URL without the release-asset path prefix must be rejected. assert not _mod._is_trusted_github_release_url( "https://objects.githubusercontent.com/abc/def", "lemonade-sdk/llamacpp-rocm", @@ -339,7 +355,7 @@ def test_lemonade_runtime_patterns_include_hip_runtime(): Lemonade ZIPs carry transitive deps (libamd_comgr, libLLVM, libclang-cpp, ...) whose names change across ROCm releases. A broad ``lib*.so*`` glob - avoids enumerating every transitive dependency by name. + avoids having to enumerate every transitive dependency by name. """ from install_llama_prebuilt import runtime_patterns_for_choice, AssetChoice @@ -353,7 +369,7 @@ def test_lemonade_runtime_patterns_include_hip_runtime(): ) pats = runtime_patterns_for_choice(choice) # The broad glob must be present so every .so in the lemonade bundle - # (including future transitive deps) gets overlaid. + # (including transitive deps added in future ROCm releases) gets overlaid. assert "lib*.so*" in pats, f"'lib*.so*' missing from linux-rocm patterns: {pats}" @@ -365,9 +381,9 @@ _pick_rocm_gfx_target = getattr(_mod, "_pick_rocm_gfx_target", None) reason = "_pick_rocm_gfx_target not present on this branch", ) def test_pick_rocm_gfx_target_honors_cuda_visible_devices(monkeypatch): - """AMD HIP honours CUDA_VISIBLE_DEVICES like HIP_VISIBLE_DEVICES; on a - gfx1151 + gfx1100 mixed host, CUDA_VISIBLE_DEVICES=1 must select gfx1100.""" - # Two GPUs; rocminfo reports each token twice (as in real tool output). + """AMD HIP honours CUDA_VISIBLE_DEVICES identically to HIP_VISIBLE_DEVICES; + on a gfx1151 + gfx1100 mixed host, CUDA_VISIBLE_DEVICES=1 must select gfx1100.""" + # Two GPUs; rocminfo reports each token twice (as in the real tool output). probe_out = "gfx1151\ngfx1151\ngfx1100\ngfx1100" monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False) monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False) @@ -396,7 +412,7 @@ def test_pick_rocm_gfx_target_same_arch_multi_gpu(monkeypatch): """Regression: [gfx1100, gfx1100, gfx1151] with HIP_VISIBLE_DEVICES=2 must return gfx1151, not fall back to GPU 0 due to dict.fromkeys collapsing the two gfx1100 entries into one and making index 2 out of range.""" - # rocminfo output for 3 GPUs (2x gfx1100 dGPU + 1x gfx1151 APU). + # Simulate rocminfo output for 3 GPUs (2x gfx1100 dGPU + 1x gfx1151 APU). # Each GPU gets its own Agent section with a few token mentions. probe_out = ( "***\nAgent 1\n***\n gfx1100 some info\n gfx1100\n" @@ -407,3 +423,96 @@ def test_pick_rocm_gfx_target_same_arch_multi_gpu(monkeypatch): monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False) monkeypatch.setenv("HIP_VISIBLE_DEVICES", "2") assert _pick_rocm_gfx_target(probe_out) == "gfx1151" + + +# --------------------------------------------------------------------------- +# Fork release scan: Windows ROCm resolves lemonade by the requested tag +# --------------------------------------------------------------------------- + +_resolve_release_asset_choice = getattr(_mod, "resolve_release_asset_choice", None) +_ApprovedReleaseChecksums = getattr(_mod, "ApprovedReleaseChecksums", None) + + +@pytest.mark.skipif( + _resolve_release_asset_choice is None or _ApprovedReleaseChecksums is None, + reason = "fork release planner not present on this branch", +) +def test_fork_scan_windows_rocm_resolves_lemonade_by_requested_tag(): + """The fork release scan pins llama_tag to per-release upstream tags + (b9457, ...) that lemonade's own tag series never contains, so the + lemonade lookup must use the requested tag ("latest") instead. Pinning + lemonade to the per-release tag 404s on every scanned release and a + Windows ROCm host ends in a rate-limited fatal instead of the lemonade + prebuilt.""" + host = _make_rocm_host("gfx1151", windows = True) + # No windows-rocm artifact in the bundle, matching current fork releases. + bundle = _rocm_bundle("gfx1151", ["gfx1151"]) + checksums = _ApprovedReleaseChecksums( + repo = "unslothai/llama.cpp", + release_tag = "v1.0", + upstream_tag = "b9457", + artifacts = {}, + ) + seen_urls: list[str] = [] + + def _fake_fetch(api_url, *args, **kwargs): + seen_urls.append(api_url) + if "lemonade-sdk" in api_url: + if api_url.endswith("/releases/latest"): + return _stub_lemonade_release() + raise RuntimeError(f"unexpected pinned lemonade fetch: {api_url}") + # ggml-org asset listing for the upstream HIP/CPU filename fallbacks. + return {"tag_name": "b9457", "assets": []} + + with patch.object(_mod, "fetch_json", side_effect = _fake_fetch): + attempts = _resolve_release_asset_choice( + host, + "b9457", # concrete per-release upstream tag from the scan loop + bundle, + checksums, + requested_tag = "latest", + ) + + lemonade = [a for a in attempts if a.source_label == "lemonade"] + assert lemonade, f"lemonade attempt missing for Windows ROCm host; got {attempts}" + assert "gfx1151" in lemonade[0].name + assert any( + u.endswith("/releases/latest") for u in seen_urls + ), f"lemonade was never resolved via /releases/latest; fetches: {seen_urls}" + assert not any( + "lemonade-sdk" in u and "/releases/tags/" in u for u in seen_urls + ), f"lemonade lookup was pinned to the fork release tag: {seen_urls}" + + +@pytest.mark.skipif( + direct_upstream_release_plan is None, + reason = "direct release planners not present on this branch", +) +def test_direct_upstream_plan_includes_lemonade_for_linux_rocm_host(): + """A Linux ROCm host on the ggml-org direct path (e.g. a --published-repo + override) must plan lemonade before the CPU tarball, mirroring the Windows + branch. The lemonade planning previously lived in the removed + --simple-policy dispatcher, so without this leg such hosts silently + install the CPU build.""" + host = _make_rocm_host("gfx1151") + release = { + "tag_name": "b9022", + "name": "b9022", + "assets": [ + { + "name": "llama-b9022-bin-ubuntu-x64.tar.gz", + "browser_download_url": ( + "https://github.com/ggml-org/llama.cpp/releases/download/" + "b9022/llama-b9022-bin-ubuntu-x64.tar.gz" + ), + } + ], + } + with patch.object(_mod, "fetch_json", return_value = _stub_lemonade_release()): + plan = direct_upstream_release_plan(release, host, "ggml-org/llama.cpp", "latest") + assert plan is not None, "Linux ROCm host should produce a direct plan" + kinds = [a.install_kind for a in plan.attempts] + sources = [a.source_label for a in plan.attempts] + assert "linux-rocm" in kinds, f"lemonade ROCm attempt missing; got {kinds}" + assert sources[0] == "lemonade", f"lemonade must be the first attempt; got {sources}" + assert "gfx1151" in plan.attempts[0].name diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index bc4fe555c1..822a32ddb8 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -121,9 +121,9 @@ def env_int( return value -# Prefer "latest" over "master": "master" bypasses the prebuilt resolver +# Prefer "latest" over "master" -- "master" bypasses the prebuilt resolver # (no matching GitHub release), forces a source build, and causes HTTP 422 -# errors. Use "master" only temporarily when the latest release lacks +# errors. Only use "master" temporarily when the latest release is missing # support for a new model architecture. DEFAULT_LLAMA_TAG = os.environ.get("UNSLOTH_LLAMA_TAG", "latest") # Default published repo for prebuilt release resolution. Linux uses @@ -145,13 +145,21 @@ LEMONADE_ROCM_RELEASES_API = f"https://api.github.com/repos/{LEMONADE_ROCM_REPO} def _lemonade_release_api_for(llama_tag: str) -> str: - """GitHub API URL for the lemonade release matching a llama.cpp tag. + """Return the GitHub API URL for the lemonade release that matches a + requested llama.cpp tag. - "latest"/unset -> /releases/latest; a pinned tag -> the same lemonade tag. - Lemonade tracks ggml-org build tags but may lag, so a tag it skipped 404s - and the caller falls back to the upstream tarball (keeps pinned installs - reproducible). Do NOT pass a fork tag -- the fork namespace always 404s. - Tag is URL-encoded (safe="") to prevent URL injection. + When llama_tag is unset or "latest", point at /releases/latest. When the + caller has pinned a specific tag (e.g. "b1260"), point at the same tag in + lemonade. Lemonade tracks `ggml-org/llama.cpp` build tags (e.g. "b1260") + but is NOT guaranteed to publish every upstream build -- lemonade may be + several builds behind ggml-org. Pinning to a specific tag that lemonade + skipped will produce a 404 and the caller falls through to the upstream + tarball; that is intentional so pinned installs stay reproducible. + Do NOT pass a `unslothai/llama.cpp` fork tag -- the fork uses its own + namespace and will always 404 against lemonade. + + The tag is URL-encoded with `safe=""` so an unexpected slash / hash / query + character cannot reshape the URL. """ normalized = (llama_tag or "").strip() if not normalized or normalized.lower() == "latest": @@ -186,90 +194,35 @@ DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS = env_int( 2, minimum = 1, ) -# Deeper macOS-only walk-back: upstream can ship a run of prebuilts built for -# a newer macOS than the host, caught only at validate time, so an older host -# must skip the whole run. Free on new hosts (first plan validates). +# Deeper macOS-only walk-back: upstream can ship a run of prebuilts built for a +# newer macOS than the host, only caught at validate time, so an older host must +# skip the whole run. Free on new hosts (first plan validates, extras unused). DEFAULT_MAX_MACOS_RELEASE_FALLBACKS = env_int( "UNSLOTH_LLAMA_MAX_MACOS_RELEASE_FALLBACKS", 16, minimum = 1, ) # Deterministic macOS pin. At b9428 ggml-org's macOS runner moved to macOS 26 -# (Tahoe), so b9428+ prebuilts load only on macOS 26+. b9415 is the last build -# stamped below 26 (arm64 minos 14, x64 minos 13.3); loads on 13.3/14/15/26. +# (Tahoe), so b9428+ prebuilts only load on macOS 26+. b9415 is the last build +# stamped below 26 (arm64 minos 14, x64 minos 13.3); loads on macOS 13.3/14/15/26. _PINNED_MACOS_FALLBACK_TAG = "b9415" _PINNED_MACOS_LATEST_FLOOR = (26, 0) FORCE_COMPILE_DEFAULT_REF = os.environ.get("UNSLOTH_LLAMA_FORCE_COMPILE_REF", "master") -# sm_103 (B300 / GB300 Blackwell Ultra) is not built natively but runs on the -# bundled base compute_100 PTX, which the driver JIT-compiles forward to sm_103. -# Listed in every bundle that ships the sm_100 build (the "newer" and -# "portable" classes) so those hosts get a prebuilt, not a source compile. -DIRECT_LINUX_BUNDLE_PROFILES: dict[str, dict[str, Any]] = { - "cuda12-older": { - "runtime_line": "cuda12", - "coverage_class": "older", - "supported_sms": ["70", "75", "80", "86", "89"], - "min_sm": 70, - "max_sm": 89, - "rank": 10, - }, - "cuda12-newer": { - "runtime_line": "cuda12", - "coverage_class": "newer", - "supported_sms": ["86", "89", "90", "100", "103", "120"], - "min_sm": 86, - "max_sm": 120, - "rank": 20, - }, - "cuda12-portable": { - "runtime_line": "cuda12", - "coverage_class": "portable", - "supported_sms": ["70", "75", "80", "86", "89", "90", "100", "103", "120"], - "min_sm": 70, - "max_sm": 120, - "rank": 30, - }, - "cuda13-older": { - "runtime_line": "cuda13", - "coverage_class": "older", - "supported_sms": ["75", "80", "86", "89"], - "min_sm": 75, - "max_sm": 89, - "rank": 40, - }, - "cuda13-newer": { - "runtime_line": "cuda13", - "coverage_class": "newer", - "supported_sms": ["86", "89", "90", "100", "103", "120"], - "min_sm": 86, - "max_sm": 120, - "rank": 50, - }, - "cuda13-portable": { - "runtime_line": "cuda13", - "coverage_class": "portable", - "supported_sms": ["75", "80", "86", "89", "90", "100", "103", "120"], - "min_sm": 75, - "max_sm": 120, - "rank": 60, - }, -} - # Lowest CUDA major we ship prebuilts for, and the highest major we probe for # installed runtime libraries. Detection and runtime-line derivation are -# generated per major so a new toolkit (cuda14, ...) needs no code change as -# long as llama.cpp keeps the cudart64_.dll / libcudart.so. naming. +# generated per major so a new toolkit (cuda14, ...) needs no code change while +# llama.cpp keeps the cudart64_.dll / libcudart.so. naming. _MIN_CUDA_MAJOR = 12 _MAX_PROBE_CUDA_MAJOR = 19 # Last ggml-org release whose Windows win-cuda-13 build is still sub-13.3 # (cuda-13.1, b9360, 2026-05-27). Upstream bumped win-cuda-13 to 13.3 at b9365 # and now ships only cuda-12.4 + cuda-13.3. cuda-12.4 predates Blackwell (ggml -# compiles sm_120 only at toolkit >= 12.8), so a Blackwell host on a -# 13.0/13.1/13.2 driver is gated off 13.3 and would drop to CPU-only 12.4. -# b9360 is immutable, so we pin its cuda-13.1 build (plus paired cudart) as a -# GPU fallback for exactly those hosts. See unslothai/unsloth#5887. +# compiles sm_120 only at toolkit >= 12.8), so a Blackwell host on a 13.0/13.1/13.2 +# driver is gated off 13.3 and would drop to a CPU-only 12.4 build. b9360 is +# immutable, so we pin its cuda-13.1 build (plus paired cudart) as a GPU +# fallback for exactly those hosts. See unslothai/unsloth#5887. _PINNED_BLACKWELL_FALLBACK_TAG = "b9360" _PINNED_BLACKWELL_FALLBACK_RUNTIME = "13.1" # Floor at 13.0: b9360 ships native sm_120a SASS (no PTX/JIT) and a bundled @@ -279,46 +232,19 @@ _PINNED_BLACKWELL_DRIVER_FLOOR = (13, 0) _BLACKWELL_MIN_SM = 120 # ggml compiles Blackwell sm_120 only at toolkit >= 12.8, so an in-release # windows-cuda build at or above this already covers Blackwell and makes the -# pinned 13.1 fallback unnecessary (cuda-12.4 is below it). +# older pinned 13.1 fallback unnecessary (cuda-12.4 is below it). _BLACKWELL_MIN_TOOLKIT = (12, 8) _PINNED_BLACKWELL_LLAMA_SHA256 = "31ddb8b42d7ab4a47cab8c48c397519f580ca502df7e73f3ab396eacc16c8e8d" _PINNED_BLACKWELL_CUDART_SHA256 = "f96935e7e385e3b2d0189239077c10fe8fd7e95690fea4afec455b1b6c7e3f18" def _cuda_runtime_lines_for_major(major: int) -> list[str]: - """Runtime lines a driver of this CUDA major can use, newest first down to - the minimum we ship. A driver runs its own major and any older one + """Runtime lines a driver of this CUDA major can use, newest major first + down to the minimum we ship. A driver runs its own major and any older one (backward compatibility).""" return [f"cuda{m}" for m in range(major, _MIN_CUDA_MAJOR - 1, -1)] -def _resolve_linux_bundle_profile(bundle_profile: str) -> "dict[str, Any] | None": - """Profile (runtime line + sm coverage) for a linux-x64-cuda- - bundle. Known majors use their published coverage; an unknown future major - reuses the newest known major's coverage for the same class as a forward - default, with the post-build GPU smoke test as backstop.""" - known = DIRECT_LINUX_BUNDLE_PROFILES.get(bundle_profile) - if known is not None: - return known - m = re.fullmatch(r"cuda(?P\d+)-(?Polder|newer|portable)", bundle_profile) - if not m: - return None - base_key = max( - ( - k - for k, v in DIRECT_LINUX_BUNDLE_PROFILES.items() - if v["coverage_class"] == m.group("klass") - ), - key = lambda k: int(re.match(r"cuda(\d+)-", k).group(1)), - default = None, - ) - if base_key is None: - return None - profile = dict(DIRECT_LINUX_BUNDLE_PROFILES[base_key]) - profile["runtime_line"] = f"cuda{m.group('major')}" - return profile - - @dataclass class HostInfo: system: str @@ -349,8 +275,8 @@ class AssetChoice: url: str source_label: str # Paired runtime archive (Windows CUDA cudart bundle). When set, - # install_from_archives also downloads it and overlays its DLLs on top - # of the main install. See unslothai/unsloth#5106. + # install_from_archives also downloads it and overlays its DLLs on + # top of the main install. See unslothai/unsloth#5106. runtime_name: str | None = None runtime_url: str | None = None runtime_sha256: str | None = None @@ -377,6 +303,10 @@ class PublishedLlamaArtifact: max_sm: int | None bundle_profile: str | None rank: int + # ROCm bundles only: the umbrella gfx target (e.g. "gfx110X") and the + # concrete gfx archs it covers (e.g. ["gfx1100", "gfx1101", ...]). + gfx_target: str | None = None + mapped_targets: list[str] = field(default_factory = list) @dataclass @@ -577,13 +507,14 @@ def is_github_api_url(url: str | None) -> bool: def is_retryable_url_error(exc: Exception) -> bool: if isinstance(exc, urllib.error.HTTPError): - # GitHub returns 403 (not 429) when the API rate limit is hit. - # Anonymous calls share a 60-req/hour bucket per runner IP, which - # CI fleets exhaust trivially. Treat 403 against api.github.com as - # retryable so we get a backoff cycle or two before the source-build - # fallback fires; sleep_backoff honours Retry-After / - # X-RateLimit-Reset for accurate waits. Real 403s on other hosts - # (private artefact downloads, auth failures) stay non-retryable. + # GitHub returns 403 (not the standard 429) when the API rate + # limit is hit. Anonymous calls share a 60-req/hour bucket per + # runner IP, which CI fleets can exhaust trivially. Treat 403 + # against api.github.com as retryable so we get one or two + # backoff cycles before the source-build fallback fires; honour + # Retry-After / X-RateLimit-Reset in sleep_backoff for accurate + # waits. Real 403s on other hosts (private artefact downloads, + # auth failures) stay non-retryable. if exc.code == 403: return is_github_api_url(getattr(exc, "url", None)) return exc.code in RETRYABLE_HTTP_STATUS @@ -602,9 +533,9 @@ _RATE_LIMIT_WAIT_CAP_SECONDS = 60.0 def _http_error_retry_delay(exc: Exception) -> float | None: """Extract a recommended wait from rate-limit headers on a 403/429. - Returns None when no header is present or the wait exceeds - _RATE_LIMIT_WAIT_CAP_SECONDS (the caller should not block then -- the - source-build fallback is faster). + Returns None when no header is present or the indicated wait is + longer than _RATE_LIMIT_WAIT_CAP_SECONDS (in which case the caller + should not block on it -- the source-build fallback is faster). """ if not isinstance(exc, urllib.error.HTTPError): return None @@ -813,12 +744,12 @@ def refs_match(candidate_ref: str | None, requested_ref: str | None) -> bool: def checkout_friendly_ref(ref_kind: str | None, ref: str | None) -> str | None: - """Normalize a source ref to a form ``git clone --branch`` accepts. + """Normalize a source ref to a form that ``git clone --branch`` accepts. - Fully qualified branch refs (``refs/heads/main``) are stripped to - ``main``; tag refs (``refs/tags/b8508``) to ``b8508``. Pull refs - (``refs/pull/123/head``) are left as-is since they are fetched - explicitly rather than cloned with ``--branch``. + Fully qualified branch refs like ``refs/heads/main`` are stripped to + ``main``; tag refs like ``refs/tags/b8508`` are stripped to ``b8508``. + Pull refs like ``refs/pull/123/head`` are left as-is since they are + always fetched explicitly rather than cloned with ``--branch``. """ if not isinstance(ref, str) or not ref: return ref @@ -868,8 +799,8 @@ def _published_windows_cuda_runtime( """Highest cuda-. published upstream that `driver` can run by default CUDA compatibility, i.e. (major, minor) <= driver. None if nothing qualifies. Gating on the driver (not just the major) keeps a 13.3 build off - a 13.1-only driver, where it would rely on the unguaranteed - minor-version-compatibility path.""" + a driver that only advertises 13.1, where it would otherwise rely on the + unguaranteed minor-version-compatibility path.""" if driver is None: return None best: int | None = None @@ -1484,7 +1415,7 @@ def direct_upstream_release_plan( ) attempts[:] = _drop_blackwell_incapable_windows_cuda(host, attempts) # Blackwell on a 13.1/13.2 driver: prefer the pinned cuda-13.1 GPU - # build over the CPU-only cuda-12.4 left by in-release gating. + # build over the CPU-only cuda-12.4 the in-release gating leaves. pinned = _pinned_windows_cuda_fallback(host, attempts) if pinned is not None: attempts.insert(0, pinned) @@ -1522,9 +1453,9 @@ def direct_upstream_release_plan( ) elif host.is_windows and host.is_arm64: # Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-win-cpu-arm64.zip - # (in the b9334 release manifest). Without this branch the selector - # returned 0 attempts and fell back to a source build on every - # Windows ARM64 host. + # (visible in the b9334 release manifest). Without this branch the + # selector returned 0 attempts and the installer fell back to a + # source build on every Windows ARM64 host. cpu_asset = f"llama-{release_tag}-bin-win-cpu-arm64.zip" cpu_url = assets.get(cpu_asset) if cpu_url: @@ -1567,6 +1498,14 @@ def direct_upstream_release_plan( ) ) elif host.is_linux and host.is_x86_64 and not host.has_usable_nvidia: + if host.has_rocm: + # Lemonade first, mirroring the Windows ROCm branch above, so a + # ROCm host routed to ggml-org does not silently get the CPU build. + lemonade_choice = resolve_lemonade_rocm_choice( + host, "ubuntu", "linux-rocm", llama_tag = requested_tag + ) + if lemonade_choice is not None: + attempts.append(lemonade_choice) asset_name = f"llama-{release_tag}-bin-ubuntu-x64.tar.gz" asset_url = assets.get(asset_name) if asset_url: @@ -1582,10 +1521,10 @@ def direct_upstream_release_plan( ) elif host.is_linux and host.is_arm64 and not host.has_usable_nvidia: # Upstream ggml-org/llama.cpp ships llama-bNNNN-bin-ubuntu-arm64.tar.gz - # (in the b9334 release manifest). Without this branch the selector - # returned 0 attempts and fell back to a source build on every Linux - # ARM64 host (DGX Spark, Ampere Altra, GitHub ubuntu-24.04-arm - # runners, etc.). + # (visible in the b9334 release manifest). Without this branch the + # selector returned 0 attempts and the installer fell back to a + # source build on every Linux ARM64 host (DGX Spark, Ampere + # Altra, GitHub-hosted ubuntu-24.04-arm runners, etc.). asset_name = f"llama-{release_tag}-bin-ubuntu-arm64.tar.gz" asset_url = assets.get(asset_name) if asset_url: @@ -1615,9 +1554,9 @@ def direct_upstream_release_plan( def pinned_macos_release_tag(host: HostInfo, repo: str) -> str | None: - """Pin b9415 (last upstream macOS build that loads below macOS 26) for a - known pre-26 host on ggml-org upstream; None keeps latest selection. The - unslothai/llama.cpp fork ships its own prebuilts (arm64 minos 14, x64 + """Pin b9415 (the last upstream macOS build that loads below macOS 26) for a + known pre-26 host on ggml-org upstream; return None to keep latest selection. + The unslothai/llama.cpp fork ships its own prebuilts (arm64 minos 14, x64 minos 13.3) and needs no pin, so this is a no-op there and for macOS 26+, unknown version, non-macOS.""" if repo != UPSTREAM_REPO: @@ -1641,20 +1580,22 @@ def resolve_simple_install_release_plans( max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS, ) -> tuple[str, list[InstallReleasePlan]]: repo = published_repo or DEFAULT_PUBLISHED_REPO - requested_tag = normalized_requested_llama_tag(llama_tag) - # The unslothai/llama.cpp fork ships only linux-x64 bundles. An arm64 Linux - # host with a GPU (GH200/GB200/DGX Spark) routes here; it must not install - # an x64 binary, so fall back to a GPU-targeting source build rather than - # the wrong arch (or silently dropping to a CPU arm64 build). - if host.is_linux and not host.is_x86_64 and repo == DEFAULT_PUBLISHED_REPO: - raise PrebuiltFallback( - f"{repo} ships only linux-x64 prebuilts; " - f"{host.machine or 'non-x64'} Linux falls back to source build" + # The fork (unslothai) ships a manifest describing every bundle's GPU/arch + # coverage, so all fork hosts select from it. Upstream (ggml-org) ships no + # manifest and is selected by asset filename in the loop below. + if repo == DEFAULT_PUBLISHED_REPO: + return _fork_manifest_release_plans( + llama_tag, + host, + published_repo, + published_release_tag, + max_release_fallbacks = max_release_fallbacks, ) + requested_tag = normalized_requested_llama_tag(llama_tag) allow_older_release_fallback = requested_tag == "latest" and not published_release_tag # macOS: pin the last upstream build that loads on a pre-26 host instead of # fetching the latest (macOS 26 only) build and walking back release by - # release. No-op on macOS 26+, unknown version, non-macOS, the fork. + # release. No-op on macOS 26+, unknown version, non-macOS, and the fork. if allow_older_release_fallback: pinned_macos = pinned_macos_release_tag(host, repo) if pinned_macos is not None: @@ -1668,10 +1609,7 @@ def resolve_simple_install_release_plans( releases = iter_release_payloads_by_time(repo, published_release_tag, requested_tag) for release in releases: try: - if host.is_linux and repo == "unslothai/llama.cpp": - plan = direct_linux_release_plan(release, host, repo, requested_tag) - else: - plan = direct_upstream_release_plan(release, host, repo, requested_tag) + plan = direct_upstream_release_plan(release, host, repo, requested_tag) if plan is None: continue except PrebuiltFallback as exc: @@ -1940,6 +1878,18 @@ def parse_published_artifact(raw: Any) -> PublishedLlamaArtifact | None: rank = int(rank_raw) except (TypeError, ValueError): raise ValueError(f"artifact {asset_name} rank was not an integer") + gfx_target_raw = raw.get("gfx_target") + gfx_target = ( + gfx_target_raw.strip() + if isinstance(gfx_target_raw, str) and gfx_target_raw.strip() + else None + ) + mapped_raw = raw.get("mapped_targets", []) + mapped_targets = ( + [value.strip() for value in mapped_raw if isinstance(value, str) and value.strip()] + if isinstance(mapped_raw, (list, tuple)) + else [] + ) return PublishedLlamaArtifact( asset_name = asset_name, install_kind = install_kind, @@ -1954,6 +1904,8 @@ def parse_published_artifact(raw: Any) -> PublishedLlamaArtifact | None: if isinstance(bundle_profile, str) and bundle_profile else None, rank = rank, + gfx_target = gfx_target, + mapped_targets = mapped_targets, ) @@ -1969,8 +1921,8 @@ def parse_published_release_bundle( if not manifest_url: return None - # Mixed repos are filtered by an explicit release-side manifest, not by - # release tag or asset filename conventions. + # Mixed repos are filtered by an explicit release-side manifest rather than + # by release tag or asset filename conventions. manifest_bytes = download_bytes( manifest_url, timeout = 30, @@ -2190,6 +2142,25 @@ def iter_published_release_bundles( yield bundle +def _artifact_covers_sms(artifact: PublishedLlamaArtifact, host_sms: Iterable[str]) -> bool: + """True when every host SM is listed in the artifact's supported_sms and + falls within its [min_sm, max_sm] range.""" + if not artifact.supported_sms or artifact.min_sm is None or artifact.max_sm is None: + return False + supported = {str(value) for value in artifact.supported_sms} + return all(sm in supported and artifact.min_sm <= int(sm) <= artifact.max_sm for sm in host_sms) + + +def _sm_range(artifact: PublishedLlamaArtifact) -> int: + """SM-coverage span used as a sort key, where a tighter (smaller) range wins. + A bundle with no SM metadata (legacy/upstream-named) gets a max range so it + sorts last and can't outrank a real targeted bundle whose tight range would + otherwise sort first.""" + if artifact.min_sm is not None and artifact.max_sm is not None: + return artifact.max_sm - artifact.min_sm + return 9999 + + def linux_cuda_choice_from_release( host: HostInfo, release: PublishedReleaseBundle, @@ -2229,8 +2200,12 @@ def linux_cuda_choice_from_release( else "none" ) ) + # arm64 CUDA hosts (DGX Spark / Grace Hopper) consume linux-arm64-cuda + # bundles; x64 hosts consume linux-cuda. The SM / runtime-line matching + # below is arch-agnostic and applies to both. + cuda_install_kind = "linux-arm64-cuda" if host.is_arm64 else "linux-cuda" published_artifacts = [ - artifact for artifact in release.artifacts if artifact.install_kind == "linux-cuda" + artifact for artifact in release.artifacts if artifact.install_kind == cuda_install_kind ] published_asset_names = sorted(artifact.asset_name for artifact in published_artifacts) selection_log.append( @@ -2281,7 +2256,7 @@ def linux_cuda_choice_from_release( url = asset_url, source_label = "published", is_ready_bundle = True, - install_kind = "linux-cuda", + install_kind = cuda_install_kind, bundle_profile = artifact.bundle_profile, runtime_line = artifact.runtime_line, coverage_class = artifact.coverage_class, @@ -2363,7 +2338,7 @@ def linux_cuda_choice_from_release( artifact, url = sorted( coverage_candidates, key = lambda item: ( - (item[0].max_sm or 0) - (item[0].min_sm or 0), + _sm_range(item[0]), item[0].rank, item[0].max_sm or 0, ), @@ -2422,11 +2397,20 @@ def validated_checksums_for_bundle( raise PrebuiltFallback( "published manifest checksum did not match the approved checksum asset" ) - # Accept bundles carrying only an exact-commit source archive - # (llama.cpp-source-commit-.tar.gz) without requiring the legacy - # llama.cpp-source-.tar.gz entry. + # Accept bundles that carry only an exact-commit source archive + # (e.g. llama.cpp-source-commit-.tar.gz) without requiring the + # legacy llama.cpp-source-.tar.gz entry. if exact_source_archive_hash(checksums) is None: require_approved_source_hash(checksums, bundle.upstream_tag) + elif source_clone_url_for_release(checksums, bundle) is None: + # No source repo in either the checksum payload or the manifest bundle: + # preferred_source_archive would silently fall back to ggml-org source at + # the upstream tag, pairing the prebuilt with a possibly mismatched tree. + # Fail closed so the resolver skips this release. + raise PrebuiltFallback( + f"approved checksum asset for {repo}@{bundle.release_tag} declared an " + "exact source archive without a source repo to clone it from" + ) return checksums @@ -2560,22 +2544,23 @@ def resolve_requested_llama_tag( Resolution order: 1. Concrete tag (e.g. "b8508") -- returned as-is. - 2. "latest" with published_repo -- the latest usable Unsloth published - bundle's upstream_tag (matches the published prebuilt metadata). - 3. "latest" without published_repo, or if (2) fails -- query upstream - ggml-org/llama.cpp. May return a newer, untested tag. + 2. "latest" with published_repo -- resolve the latest usable Unsloth + published release bundle and return its upstream_tag. This is the + preferred version that matches the published prebuilt metadata. + 3. "latest" without published_repo or if (2) fails -- query the upstream + ggml-org/llama.cpp repo. This may return a newer, untested tag. - The Unsloth repo is preferred because its releases are pinned to upstream - tags validated with Unsloth Studio; the upstream bleeding-edge tag risks - API/ABI incompatibilities. + The Unsloth repo is preferred because its releases are pinned to specific + upstream tags that have been validated with Unsloth Studio. Using the + upstream bleeding-edge tag risks API/ABI incompatibilities. """ normalized_requested = normalized_requested_llama_tag(requested_tag) if normalized_requested != "latest": return normalized_requested # Prefer the Unsloth release repo tag (tested/approved) over bleeding-edge - # upstream. E.g. unslothai/llama.cpp may publish b8508 while ggml-org - # latest is b8514. The source-build fallback should compile the same - # version the prebuilt path would have installed. + # upstream. For example, unslothai/llama.cpp may publish b8508 while + # ggml-org/llama.cpp latest is b8514. The source-build fallback should + # compile the same version the prebuilt path would have installed. if published_repo: try: return resolve_published_release( @@ -2585,7 +2570,7 @@ def resolve_requested_llama_tag( ).bundle.upstream_tag except Exception: pass - # Fall back to the upstream ggml-org latest release tag + # Fall back to upstream ggml-org latest release tag return latest_upstream_release_tag() @@ -2607,8 +2592,16 @@ def exact_source_archive_hash(checksums: ApprovedReleaseChecksums) -> ApprovedAr return checksums.artifacts.get(exact_source_archive_logical_name(checksums.source_commit)) -def source_clone_url_from_checksums(checksums: ApprovedReleaseChecksums) -> str | None: - return source_repo_clone_url(checksums.source_repo, checksums.source_repo_url) +def source_clone_url_for_release( + checksums: ApprovedReleaseChecksums, bundle: PublishedReleaseBundle +) -> str | None: + # Single source of truth for "where do we clone source from", shared by the + # validation gate and source_build_plan_for_release: take the repo from the + # checksum payload, falling back field by field to the manifest bundle. + return source_repo_clone_url( + checksums.source_repo or bundle.source_repo, + checksums.source_repo_url or bundle.source_repo_url, + ) def source_build_plan_for_release(release: ResolvedPublishedRelease) -> SourceBuildPlan: @@ -2620,7 +2613,7 @@ def source_build_plan_for_release(release: ResolvedPublishedRelease) -> SourceBu resolved_source_ref = checksums.resolved_source_ref or release.bundle.resolved_source_ref source_commit = checksums.source_commit or release.bundle.source_commit source_ref_kind = checksums.source_ref_kind or release.bundle.source_ref_kind - source_url = source_repo_clone_url(source_repo, source_repo_url) + source_url = source_clone_url_for_release(checksums, release.bundle) if exact_source is not None and source_url and source_commit: return SourceBuildPlan( source_url = source_url, @@ -2741,38 +2734,38 @@ def _pick_rocm_gfx_target(out: str) -> str | None: A bare first-match picked the wrong device on mixed APU + dGPU hosts (e.g. Strix Halo gfx1151 + discrete RX 7900 gfx1100). Respect HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES / CUDA_VISIBLE_DEVICES so the - asset matches what HIP runs on; falls back to the first GPU with no env - var set. + asset matches what HIP actually runs on. Falls back to the first GPU when + no env var is set. rocminfo / hipinfo print the same gfx token multiple times per GPU (Name, - ISA, marketing-name). We first split the output on per-GPU section headers - (rocminfo: "Agent N" blocks, hipinfo: "device#N" entries) and take exactly - one gfx token per section -- this gives the correct per-GPU list even on - same-arch multi-GPU hosts (e.g. two RX 7900 XTX) where global dict.fromkeys - dedup would collapse both to one entry and make HIP_VISIBLE_DEVICES=1 point - out of range. + ISA, marketing-name). We first try to split the output on per-GPU section + headers (rocminfo: "Agent N" blocks, hipinfo: "device#N" entries) and take + exactly one gfx token per section. This gives the correct per-GPU list even + on same-arch multi-GPU hosts (e.g. two RX 7900 XTX cards) where global + dict.fromkeys dedup would collapse both cards to a single entry and make + HIP_VISIBLE_DEVICES=1 point out of range. Falls back to insertion-order dedup when the output has no recognisable section markers (flat gfx-string inputs, unit-test stubs, etc.). Empty / "-1" env values mean no AMD GPU is visible to HIP: return None. """ - # Build a per-GPU token list by splitting on section boundaries. rocminfo - # sections start with "Agent N" lines (optionally between rows of - # asterisks); hipinfo sections with "device#N". + # Try to build a per-GPU token list by splitting on section boundaries. + # rocminfo sections are introduced by "Agent N" lines (optionally between + # rows of asterisks). hipinfo sections start with "device#N". _sections = re.split( r"(?mi)^\s*\*+\s*$\s*agent\s+\d+\s*$|\bdevice\s*#\s*\d+\b", out, ) if len(_sections) > 1: - # One gfx token per GPU section preserves physical order. + # Section-based: one gfx token per GPU section preserves physical order. _tokens: list[str] = [] for _sec in _sections[1:]: _m = re.search(r"gfx[1-9][0-9a-z]{2,3}", _sec.lower()) if _m: _tokens.append(_m.group(0)) else: - # Fallback: insertion-order dedup (flat strings / unknown formats). + # Fallback: insertion-order dedup (handles flat strings / unknown formats). _raw = re.findall(r"gfx[1-9][0-9a-z]{2,3}", out.lower()) _tokens = list(dict.fromkeys(_raw)) @@ -2788,7 +2781,7 @@ def _pick_rocm_gfx_target(out: str) -> str | None: break if _vis_raw is not None: _vis = _vis_raw.strip() - # Empty or "-1" means "no AMD GPU visible" (matches the rest of Studio) + # Empty or "-1" means "no AMD GPU visible" (matches the rest of Studio). if _vis == "" or _vis == "-1": return None _first = _vis.split(",")[0].strip() @@ -2820,10 +2813,11 @@ def detect_host() -> HostInfo: has_physical_nvidia = False has_usable_nvidia = False if nvidia_smi: - # Require `nvidia-smi -L` to list a GPU before treating the host as - # NVIDIA. The "NVIDIA-SMI ..." banner prints even when the command - # can't reach the driver (e.g. stale container leftovers), which - # would misclassify an AMD ROCm host as NVIDIA and skip the ROCm path. + # Require `nvidia-smi -L` to actually list a GPU before treating the + # host as NVIDIA. The banner text "NVIDIA-SMI ..." is printed even + # when the command fails to communicate with the driver (e.g. stale + # container leftovers), which would otherwise misclassify an AMD + # ROCm host as NVIDIA and short-circuit the ROCm path. try: listing = run_capture([nvidia_smi, "-L"], timeout = 20) gpu_lines = [line for line in listing.stdout.splitlines() if line.startswith("GPU ")] @@ -2836,9 +2830,9 @@ def detect_host() -> HostInfo: try: result = run_capture([nvidia_smi], timeout = 20) merged = "\n".join(part for part in (result.stdout, result.stderr) if part) - # Newer NVIDIA drivers (e.g. 610.x on Windows) print "CUDA UMD - # Version: X.Y" instead of the legacy "CUDA Version: X.Y"; accept - # both spellings. + # Newer NVIDIA drivers (e.g. 610.x on Windows) print + # "CUDA UMD Version: X.Y" instead of the legacy + # "CUDA Version: X.Y"; accept both spellings. cuda_match = re.search( r"CUDA(?: UMD)? Version:\s*(\d+)\.(\d+)", merged, @@ -2881,10 +2875,10 @@ def detect_host() -> HostInfo: if visible_gpu_rows: has_usable_nvidia = True - # Older nvidia-smi (pre -L support) hits the except in the - # first try block but still succeeds here, leaving - # has_physical_nvidia unset. Mirror the -L path so downstream - # diagnostics on line ~4390 still run. + # Older nvidia-smi versions (pre -L support) hit the + # except in the first try block but still succeed here, + # leaving has_physical_nvidia unset. Mirror the -L path + # so downstream diagnostics on line ~4390 still run. if not has_physical_nvidia: has_physical_nvidia = True elif visible_device_tokens == []: @@ -2913,10 +2907,10 @@ def detect_host() -> HostInfo: _dxg_probe_env = {**os.environ} _dxg_probe_env.setdefault("HSA_ENABLE_DXG_DETECTION", "1") for _cmd, _check in ( - # rocminfo: a real gfx GPU id (3-4 chars, nonzero first digit). - # gfx000 is the CPU agent; ROCm 6.1+ also emits generic ISA lines - # ("gfx11-generic", "gfx9-4-generic") with only 1-2 digits before - # the dash, which must not be treated as a real GPU. + # rocminfo: look for a real gfx GPU id (3-4 chars, nonzero first digit). + # gfx000 is the CPU agent; ROCm 6.1+ also emits generic ISA lines like + # "gfx11-generic" or "gfx9-4-generic" which only have 1-2 digits before + # the dash and must not be treated as a real GPU. ( ["rocminfo"], lambda out: bool(re.search(r"gfx[1-9][0-9a-z]{2,3}", out.lower())), @@ -2944,13 +2938,14 @@ def detect_host() -> HostInfo: rocm_gfx_target = _pick_rocm_gfx_target(_result.stdout) break elif is_windows: - # Windows: prefer active probes that validate GPU presence. hipinfo / - # amd-smi are often NOT on PATH -- the HIP SDK installer sets HIP_PATH - # / ROCM_PATH but doesn't always add the bin dir to PATH. Mirror - # setup.ps1's fallback: check the env-var bin dirs before giving up so - # `has_rocm` isn't silently False when PATH isn't updated yet. + # Windows: prefer active probes that validate GPU presence. + # hipinfo / amd-smi are often NOT on PATH -- the HIP SDK installer + # sets HIP_PATH / ROCM_PATH but does not always add the bin dir to + # the system PATH. Mirror setup.ps1's fallback: check the env-var + # bin dirs before giving up so that `has_rocm` is not silently False + # on machines where the PATH is not yet updated. def _resolve_exe(name: str) -> str | None: - """Full path to `name`, checking PATH then HIP_PATH/ROCM_PATH bin.""" + """Return full path to `name`, checking PATH then HIP_PATH/ROCM_PATH bin.""" found = shutil.which(name) if found: return found @@ -2987,8 +2982,8 @@ def detect_host() -> HostInfo: # hipinfo reports "gcnArchName: gfx1100" -- extract if present rocm_gfx_target = _pick_rocm_gfx_target(_result.stdout) break - # Note: amdhip64.dll presence alone is NOT GPU evidence -- the HIP SDK - # can be installed without an AMD GPU. + # Note: amdhip64.dll presence alone is NOT treated as GPU evidence + # since the HIP SDK can be installed without an AMD GPU. return HostInfo( system = system, @@ -3013,7 +3008,7 @@ def detect_host() -> HostInfo: def _normalize_forwarded_gfx(value: str | None) -> str | None: """Extract a single gfx token from a forwarded --rocm-gfx / env value. setup.sh/setup.ps1 already picked the active GPU, so take the token as-is - without re-applying visible-device selection. Ignore malformed input.""" + without re-applying visible-device selection. Ignore anything malformed.""" if not value: return None m = re.search(r"gfx[1-9][0-9a-z]{2,3}", value.lower()) @@ -3029,8 +3024,8 @@ def _apply_host_overrides( ) -> HostInfo: """Fold setup.sh/setup.ps1's forwarded detection into the host profile. A forwarded gfx (--rocm-gfx or UNSLOTH_ROCM_GFX_ARCH) is authoritative and - implies ROCm: the installer's own hipinfo/amd-smi probe can miss the arch - on amd-smi-only hosts or when setup inferred it from the GPU name, leaving + implies ROCm: the installer's own hipinfo/amd-smi probe can miss the arch on + amd-smi-only hosts or when setup inferred it from the GPU name, leaving rocm_gfx_target None and no lemonade prebuilt selected. force_cpu is the opposite explicit signal (arm64 Linux GPU host whose source build failed): drop GPU attributes so the CPU prebuilt for this OS/arch is selected.""" @@ -3072,7 +3067,7 @@ def compatible_linux_runtime_lines(host: HostInfo) -> list[str]: def windows_runtime_line_info() -> dict[str, tuple[str, ...]]: # Generated per CUDA major (newest first) so a new toolkit is detected - # without code changes while the cudart64_.dll naming holds. + # without a code change while the cudart64_.dll naming holds. return { f"cuda{m}": ( f"cudart64_{m}*.dll", @@ -3098,9 +3093,10 @@ def detected_windows_runtime_lines() -> tuple[list[str], dict[str, list[str]]]: def compatible_windows_runtime_lines(host: HostInfo) -> list[str]: if not host.driver_cuda_version: return [] - major, minor = host.driver_cuda_version - # cuda12 prebuilts need a 12.4+ driver; cuda13+ any minor. - if major < _MIN_CUDA_MAJOR or (major == _MIN_CUDA_MAJOR and minor < 4): + major, _minor = host.driver_cuda_version + # cuda12 app bundles are toolkit-12.8 builds with bundled runtime libs; CUDA + # minor-version compatibility runs them on any 12.x driver, same as Linux. + if major < _MIN_CUDA_MAJOR: return [] return _cuda_runtime_lines_for_major(major) @@ -3239,8 +3235,8 @@ def windows_cuda_attempts( runtime_order.extend( runtime_line for runtime_line in normal_runtime_lines if runtime_line not in runtime_order ) - # Keep every driver-compatible line reachable as a fallback, so a line - # gated out by driver version still drops to an older major (cuda13->cuda12). + # Keep every driver-compatible line reachable as a fallback, so a line gated + # out by the driver version still drops to an older major (cuda13 -> cuda12). runtime_order.extend( runtime_line for runtime_line in compatible_runtime_lines @@ -3259,7 +3255,7 @@ def windows_cuda_attempts( for runtime_line in runtime_order: major = int(runtime_line.removeprefix("cuda")) # Track whatever minor llama.cpp actually ships for this major - # (cuda13 -> 13.1, 13.3, ...). Skip the line when the release lacks a + # (cuda13 -> 13.1, 13.3, ...). Skip the line when the release has no # matching asset instead of guessing a now-missing name. runtime = _published_windows_cuda_runtime(upstream_assets, major, host.driver_cuda_version) if runtime is None: @@ -3280,10 +3276,10 @@ def windows_cuda_attempts( + ",".join(windows_cuda_upstream_asset_names(llama_tag, runtime)) ) continue - # Pair the cudart bundle when upstream ships it; otherwise the binary - # needs a system CUDA toolkit on PATH at runtime (#5106). Only pair - # when the selected main archive is the binary archive, not the cudart - # archive itself. + # Pair the cudart bundle when upstream ships it. Without this + # the binary needs a system CUDA toolkit on PATH at runtime + # (#5106). Only pair when the selected main archive is the + # binary archive, not the cudart archive itself. runtime_archive_name: str | None = None runtime_archive_url: str | None = None if selected_name.startswith("llama-"): @@ -3322,23 +3318,19 @@ def windows_cuda_attempts( def _windows_cuda_attempt_covers_blackwell(attempt: AssetChoice) -> bool: - """True if an in-release windows-cuda attempt covers Blackwell sm_120. - - Manifest-backed app bundles carry their compiled SM list, so trust that - first (the published cuda12 bundles are toolkit-12.8 builds that include - sm_120 even though their runtime line says cuda12). Upstream ggml-org - zips have no manifest metadata; infer from the CUDA minor in the asset - name (sm_120 needs toolkit >= 12.8).""" + """True if an in-release windows-cuda attempt yields a Blackwell sm_120 + capable build. The fork's app-named bundles declare their SM coverage + directly; legacy upstream-named bundles instead encode their CUDA toolkit + minor in the filename (covers Blackwell at toolkit >= 12.8).""" if attempt.install_kind != "windows-cuda": return False - if attempt.max_sm is not None: - return attempt.max_sm >= _BLACKWELL_MIN_SM - if attempt.supported_sms: - sms = normalize_compute_caps(attempt.supported_sms) - if sms: - return int(sms[-1]) >= _BLACKWELL_MIN_SM + # Legacy upstream-named bundles encode their toolkit minor; it is the binding + # constraint (a 12.4 toolkit cannot offload sm_120 whatever its metadata says). m = re.search(r"-bin-win-cuda-(\d+)\.(\d+)-x64\.zip$", attempt.name) - return m is not None and (int(m.group(1)), int(m.group(2))) >= _BLACKWELL_MIN_TOOLKIT + if m is not None: + return (int(m.group(1)), int(m.group(2))) >= _BLACKWELL_MIN_TOOLKIT + # App-named bundles carry no minor and declare their SM coverage directly. + return attempt.max_sm is not None and attempt.max_sm >= _BLACKWELL_MIN_SM def _host_is_blackwell(host: HostInfo) -> bool: @@ -3369,16 +3361,16 @@ def _pinned_windows_cuda_fallback( host: HostInfo, existing_cuda_attempts: list[AssetChoice] ) -> AssetChoice | None: """Pinned GPU fallback for a Blackwell host the in-release build gates off. - Upstream stopped publishing a sub-13.3 Windows cuda13 build after b9360, - and cuda-12.4 cannot offload sm_120, so a 13.1/13.2 driver would land on - CPU. b9360's cuda-13.1 build is immutable and runs on those drivers. - Returns None (dormant) whenever the in-release selection already offers a - Blackwell-capable build (toolkit >= 12.8, e.g. a runnable cuda13/cuda14), - so it self-disables once upstream ships a driver-runnable build again. + Upstream stopped publishing a sub-13.3 Windows cuda13 build after b9360, and + cuda-12.4 cannot offload sm_120, so a 13.1/13.2 driver would land on CPU. + b9360's cuda-13.1 build is immutable and runs on those drivers. Returns None + (dormant) whenever the in-release selection already offers a Blackwell-capable + build (toolkit >= 12.8, e.g. a runnable cuda13/cuda14), so it self-disables + once upstream ships a driver-runnable build again. - The b9360 binary reuses the current release's source tree and convert - scripts and is recorded via binary_release_tag, the same binary/source - split used for the lemonade prebuilt.""" + The b9360 binary reuses the current release's source tree and convert scripts + and is recorded via binary_release_tag, the same binary/source split used for + the lemonade prebuilt.""" if not (host.is_windows and host.is_x86_64 and host.has_usable_nvidia): return None driver = host.driver_cuda_version @@ -3420,8 +3412,8 @@ def _pinned_windows_cuda_fallback( def _augment_checksums_with_pin( checksums: ApprovedReleaseChecksums, pin: AssetChoice ) -> ApprovedReleaseChecksums: - """Add the pin's verified hashes to a copy of the approved checksums so - apply_approved_hashes keeps it on the published path (b9360 isn't in the + """Add the pin's own verified hashes to a copy of the approved checksums so + apply_approved_hashes keeps it on the published path (b9360 is not in the release manifest).""" artifacts = dict(checksums.artifacts) if pin.expected_sha256: @@ -3445,7 +3437,7 @@ def _with_pinned_windows_cuda_fallback( host: HostInfo, attempts: list[AssetChoice], checksums: ApprovedReleaseChecksums ) -> tuple[list[AssetChoice], ApprovedReleaseChecksums]: """Insert the Blackwell pin ahead of the Windows CUDA attempts and keep it - through apply_approved_hashes, or return inputs unchanged when dormant. + through apply_approved_hashes, or return the inputs unchanged when dormant. Gives the published install path the same GPU fallback as the simple path.""" attempts = _drop_blackwell_incapable_windows_cuda(host, attempts) pin = _pinned_windows_cuda_fallback(host, attempts) @@ -3461,30 +3453,6 @@ def published_windows_cuda_attempts( selection_preamble: Iterable[str] = (), ) -> list[AssetChoice]: selection_log = list(release.selection_log) + list(selection_preamble) - # Seed the runtime-line ordering from the real published windows-cuda minors - # (their names encode the minor), so a future CUDA major published here is - # ordered too rather than a hardcoded cuda12/cuda13 pair. Keys mirror the - # upstream naming so windows_cuda_attempts can match them; fall back to the - # long-standing default when the release lists no windows-cuda asset. - published_minors: list[str] = [] - for artifact in release.artifacts: - if artifact.install_kind != "windows-cuda": - continue - m = re.search(r"-bin-win-cuda-(\d+\.\d+)-x64\.zip$", artifact.asset_name) - if m: - published_minors.append(m.group(1)) - if not published_minors: - published_minors = ["12.4", "13.1"] - runtime_order = windows_cuda_attempts( - host, - release.upstream_tag, - { - f"llama-{release.upstream_tag}-bin-win-cuda-{minor}-x64.zip": "published" - for minor in published_minors - }, - preferred_runtime_line, - selection_log, - ) published_artifacts = [ artifact for artifact in release.artifacts if artifact.install_kind == "windows-cuda" ] @@ -3494,61 +3462,147 @@ def published_windows_cuda_attempts( continue artifacts_by_runtime.setdefault(artifact.runtime_line, []).append(artifact) + # Order the runtime lines to try. Legacy upstream-named bundles encode a CUDA + # minor in the filename and are driver-gated per minor. The fork's app-named + # bundles carry no minor: their runtime line (cuda12/cuda13) is gated at the + # CUDA *major* level (a 13.0 driver runs any cuda13 build) and by what torch + # actually provides on the host, preferring the torch line. Routing them + # through the synthetic-minor path wrongly dropped cuda13 on a 13.0 driver. + legacy_minors: list[str] = [] + for artifact in published_artifacts: + m = re.search(r"-bin-win-cuda-(\d+\.\d+)-x64\.zip$", artifact.asset_name) + if m: + legacy_minors.append(m.group(1)) + if legacy_minors: + ordered_lines = [ + attempt.runtime_line + for attempt in windows_cuda_attempts( + host, + release.upstream_tag, + { + f"llama-{release.upstream_tag}-bin-win-cuda-{minor}-x64.zip": "published" + for minor in legacy_minors + }, + preferred_runtime_line, + selection_log, + ) + if attempt.runtime_line + ] + else: + detected, _ = detected_windows_runtime_lines() + compatible = compatible_windows_runtime_lines(host) + # Prefer lines whose runtime DLLs are on disk, but fall back to the + # driver-derived order when none are detected (Windows torch bundles + # cudart in torch/lib, which probing misses) or when detected DLLs are + # incompatible with the driver. The app bundle ships its own runtime, so + # the driver major is the real constraint. Mirrors the legacy + # windows_cuda_attempts fallback; without it a torch-only host gets no + # fork attempt and silently drops to the upstream build. + ordered_lines = [line for line in compatible if line in detected] or list(compatible) + if preferred_runtime_line and preferred_runtime_line in ordered_lines: + ordered_lines = [preferred_runtime_line] + [ + line for line in ordered_lines if line != preferred_runtime_line + ] + selection_log.append( + "windows_cuda_selection: app-bundle runtime lines (major-gated)=" + + (",".join(ordered_lines) if ordered_lines else "none") + ) + + host_sms = normalize_compute_caps(host.compute_caps) attempts: list[AssetChoice] = [] - for ordered_attempt in runtime_order: - runtime_line = ordered_attempt.runtime_line + for runtime_line in ordered_lines: if not runtime_line: continue - candidates = sorted( - artifacts_by_runtime.get(runtime_line, []), - key = lambda artifact: (artifact.rank, artifact.asset_name), - ) - for artifact in candidates: + # Pick the artifact whose SM coverage fits the host, preferring the + # tightest targeted bundle and falling back to portable -- the same + # policy as linux_cuda_choice_from_release. Without this, app-named + # bundles (no minor in the filename) skip the SM filter and the + # lowest-rank "older" bundle is chosen for every host, breaking newer + # GPUs (e.g. Blackwell sm120 on a cuda12-older bundle capped at sm89). + targeted: list[tuple[PublishedLlamaArtifact, str, re.Match[str] | None]] = [] + portable: tuple[PublishedLlamaArtifact, str, re.Match[str] | None] | None = None + for artifact in artifacts_by_runtime.get(runtime_line, []): asset_url = release.assets.get(artifact.asset_name) if not asset_url: continue am = re.search(r"-bin-win-cuda-(\d+)\.(\d+)-x64\.zip$", artifact.asset_name) - # Gate the published minor against the driver so it can never - # bypass the driver-version gate. + # Legacy upstream-named bundles encode the minor; gate it against the + # driver. app-named bundles carry no minor and are driver-gated at the + # runtime-line level by windows_cuda_attempts above. if ( am is not None and host.driver_cuda_version is not None and (int(am.group(1)), int(am.group(2))) > host.driver_cuda_version ): continue - # See windows_cuda_attempts: pair the cudart bundle for the real minor. - runtime_archive_name: str | None = None - runtime_archive_url: str | None = None - if am is not None and artifact.asset_name.startswith("llama-"): - runtime = f"{am.group(1)}.{am.group(2)}" - cudart_name = f"cudart-llama-bin-win-cuda-{runtime}-x64.zip" - cudart_url = release.assets.get(cudart_name) - if cudart_url and cudart_url != asset_url: - runtime_archive_name = cudart_name - runtime_archive_url = cudart_url - attempt_log = list(ordered_attempt.selection_log or []) + [ - "windows_cuda_selection: selected published asset " - f"{artifact.asset_name} for runtime_line={runtime_line}" - ] - if runtime_archive_name: - attempt_log.append( - f"windows_cuda_selection: paired published runtime archive {runtime_archive_name}" - ) - attempts.append( - AssetChoice( - repo = release.repo, - tag = release.release_tag, - name = artifact.asset_name, - url = asset_url, - source_label = "published", - install_kind = "windows-cuda", - runtime_line = runtime_line, - runtime_name = runtime_archive_name, - runtime_url = runtime_archive_url, - selection_log = attempt_log, - ) + # Only SM-filter artifacts that declare full SM metadata (the app + # bundles). Legacy/upstream-named artifacts without it keep the old + # rank-based selection rather than being dropped. + has_sm_info = ( + bool(artifact.supported_sms) + and artifact.min_sm is not None + and artifact.max_sm is not None ) - break + if host_sms and has_sm_info and not _artifact_covers_sms(artifact, host_sms): + continue + if not host_sms and has_sm_info and artifact.coverage_class != "portable": + continue + if artifact.coverage_class == "portable": + portable = (artifact, asset_url, am) + else: + targeted.append((artifact, asset_url, am)) + chosen: tuple[PublishedLlamaArtifact, str, re.Match[str] | None] | None = None + if targeted: + chosen = sorted( + targeted, + key = lambda item: ( + _sm_range(item[0]), + item[0].rank, + item[0].max_sm or 0, + ), + )[0] + elif portable is not None: + chosen = portable + if chosen is None: + continue + artifact, asset_url, am = chosen + # See windows_cuda_attempts: pair the cudart bundle for the real minor. + runtime_archive_name: str | None = None + runtime_archive_url: str | None = None + if am is not None and artifact.asset_name.startswith("llama-"): + runtime = f"{am.group(1)}.{am.group(2)}" + cudart_name = f"cudart-llama-bin-win-cuda-{runtime}-x64.zip" + cudart_url = release.assets.get(cudart_name) + if cudart_url and cudart_url != asset_url: + runtime_archive_name = cudart_name + runtime_archive_url = cudart_url + attempt_log = list(selection_log) + [ + "windows_cuda_selection: selected published asset " + f"{artifact.asset_name} for runtime_line={runtime_line}" + ] + if runtime_archive_name: + attempt_log.append( + f"windows_cuda_selection: paired published runtime archive {runtime_archive_name}" + ) + attempts.append( + AssetChoice( + repo = release.repo, + tag = release.release_tag, + name = artifact.asset_name, + url = asset_url, + source_label = "published", + install_kind = "windows-cuda", + runtime_line = runtime_line, + runtime_name = runtime_archive_name, + runtime_url = runtime_archive_url, + bundle_profile = artifact.bundle_profile, + coverage_class = artifact.coverage_class, + supported_sms = artifact.supported_sms, + min_sm = artifact.min_sm, + max_sm = artifact.max_sm, + selection_log = attempt_log, + ) + ) return attempts @@ -3610,9 +3664,9 @@ def _detect_host_rocm_version() -> tuple[int, int] | None: """Return (major, minor) of the installed ROCm runtime, or None. Best-effort read from /opt/rocm/.info/version, amd-smi version, and - hipconfig --version. Used to pick a compatible upstream llama.cpp ROCm - prebuilt rather than the numerically newest one (which can be newer than - the host runtime). + hipconfig --version. Used to pick a compatible upstream llama.cpp + ROCm prebuilt rather than always taking the numerically newest one + (which can be newer than the host runtime). """ rocm_root = os.environ.get("ROCM_PATH") or "/opt/rocm" for path in ( @@ -3622,9 +3676,9 @@ def _detect_host_rocm_version() -> tuple[int, int] | None: try: with open(path) as fh: parts = fh.read().strip().split("-")[0].split(".") - # Explicit length guard so we don't rely on the broad except - # below to swallow IndexError when the version file has a single - # component (e.g. "6\n" on a partial install). + # Explicit length guard avoids relying on the broad except + # below to swallow IndexError when the version file contains + # a single component (e.g. "6\n" on a partial install). if len(parts) >= 2: return int(parts[0]), int(parts[1]) except Exception: @@ -3661,8 +3715,8 @@ def _detect_host_rocm_version() -> tuple[int, int] | None: # Distro package-manager fallbacks. Mirrors install.sh::get_torch_index_url # and _detect_rocm_version() in install_python_stack.py so package-managed - # ROCm hosts without /opt/rocm/.info/version still report a usable version, - # letting the <= host version filter in resolve_upstream_asset_choice pick + # ROCm hosts without /opt/rocm/.info/version still report a usable version + # and the <= host version filter in resolve_upstream_asset_choice picks # the correct upstream prebuilt instead of the newest-regardless fallback. for _cmd in ( ["dpkg-query", "-W", "-f=${Version}\n", "rocm-core"], @@ -3684,7 +3738,7 @@ def _detect_host_rocm_version() -> tuple[int, int] | None: if _result.returncode != 0 or not _result.stdout.strip(): continue _raw = _result.stdout.strip() - # dpkg can prepend an epoch ("1:6.3.0-1"); strip it first. + # dpkg can prepend an epoch ("1:6.3.0-1"); strip it before parsing. _raw = re.sub(r"^\d+:", "", _raw) _m = re.match(r"(\d+)[.-](\d+)", _raw) if _m: @@ -3693,7 +3747,7 @@ def _detect_host_rocm_version() -> tuple[int, int] | None: # Map detected gfx IDs to lemonade-sdk asset family suffixes. -# More-specific prefixes must precede shorter ones (e.g. gfx1151 before gfx110). +# More-specific prefixes must come before shorter ones (e.g. gfx1151 before gfx110). _LEMONADE_GFX_FAMILIES: list[tuple[str, str]] = [ ("gfx1151", "gfx1151"), ("gfx1150", "gfx1150"), @@ -3711,14 +3765,56 @@ def _lemonade_gfx_family(gfx_id: str) -> str | None: return None +def published_rocm_choice_for_host( + release: PublishedReleaseBundle, host: HostInfo, install_kind: str +) -> AssetChoice | None: + """Select the published ROCm bundle whose gfx target covers the host GPU. + + The manifest's gfx_target uses the same umbrella family labels that + _lemonade_gfx_family produces (gfx110X, gfx120X, ...), so the host's detected + gfx is matched either to that family or to the bundle's concrete + mapped_targets list. Returns None when no published bundle covers the GPU, so + the caller can fall back (lemonade / upstream HIP).""" + if not host.rocm_gfx_target: + return None + gfx = host.rocm_gfx_target.lower().strip() + for artifact in release.artifacts: + if artifact.install_kind != install_kind: + continue + # Match on the concrete built-arch list, not the family prefix: an + # in-generation-but-unbuilt arch (e.g. gfx1033 in the gfx103 prefix) must + # NOT be served the family bundle. None makes the caller fall back to a + # source build for that GPU. + if gfx not in {target.lower() for target in artifact.mapped_targets}: + continue + asset_url = release.assets.get(artifact.asset_name) + if not asset_url: + continue + return AssetChoice( + repo = release.repo, + tag = release.release_tag, + name = artifact.asset_name, + url = asset_url, + source_label = "published", + install_kind = install_kind, + selection_log = list(release.selection_log) + + [ + f"rocm_selection: gpu={host.rocm_gfx_target} " + f"selected published {artifact.asset_name}" + ], + ) + return None + + def _is_trusted_github_release_url(url: str, expected_repo: str) -> bool: """Validate a release asset URL points at GitHub's expected hosts. Accepts: https://github.com/{expected_repo}/releases/download/... https://objects.githubusercontent.com/... (GitHub's release CDN) - Anything else (http://, raw.githubusercontent.com, gist, etc.) is rejected - so a malicious API response can't redirect downloads to an attacker host. + Anything else (including http://, raw.githubusercontent.com, gist, etc.) + is rejected so a malicious API response cannot redirect downloads to an + attacker-chosen host. """ if not isinstance(url, str) or not url: return False @@ -3731,8 +3827,8 @@ def _is_trusted_github_release_url(url: str, expected_repo: str) -> bool: host = (parsed.netloc or "").lower() if host == "objects.githubusercontent.com": # GitHub's release CDN. Restrict to release-asset paths so a tampered - # API response pointing at an arbitrary CDN object is rejected. Real - # release asset URLs carry the "/github-production-release-asset-" + # API response pointing at an arbitrary CDN object is still rejected. + # Real release asset URLs carry the "/github-production-release-asset-" # prefix; gist / raw / avatar CDN paths do not. return parsed.path.startswith("/github-production-release-asset-") if host == "github.com": @@ -3744,12 +3840,12 @@ def _is_trusted_github_release_url(url: str, expected_repo: str) -> bool: def _fetch_lemonade_release_cached(api_url: str, llama_tag: str) -> "dict | None": """Cached wrapper around fetch_json for lemonade release lookups. - resolve_lemonade_rocm_choice() is called twice per install (direct planner - + resolve_upstream_asset_choice) with identical arguments. Without - memoisation each install hits api.github.com twice, doubling the - rate-limit failure surface on busy CI runners. Cache is process-scoped; - tests that vary fetch_json's return value across calls should call - cache_clear(). + resolve_lemonade_rocm_choice() is called twice per install (once from the + direct planner, once from resolve_upstream_asset_choice) with identical + arguments. Without memoisation, each install hits api.github.com twice, + doubling the rate-limit failure surface on busy CI runners. Cache is + process-scoped; tests that need to vary fetch_json's return value across + invocations should call cache_clear(). """ try: return fetch_json(api_url) @@ -3775,21 +3871,23 @@ def resolve_lemonade_rocm_choice( os_prefix: lemonade's asset filename label, NOT a host-distro filter. Pass "ubuntu" for any Linux host (Arch, Fedora, openSUSE, - Debian, ...) -- lemonade publishes one Linux variant, a - manylinux-style glibc build that runs on any distro with a - recent-enough glibc. Pass "windows" for Windows hosts. + Debian, ...) -- lemonade only publishes one Linux variant + and it is a manylinux-style glibc build that runs on any + distro with a recent-enough glibc. Pass "windows" for + Windows hosts. install_kind: "linux-rocm" or "windows-hip" llama_tag: the requested upstream llama.cpp tag ("latest" or a pinned - release like "b1260"). When pinned, fetch the matching - lemonade release; if lemonade hasn't published that tag, skip - silently (caller falls through to upstream) rather than drift - to whatever lemonade ships as latest. + release like "b1260"). When pinned, the resolver fetches + the matching lemonade release. When the pinned tag is not + published by lemonade we skip silently (and the caller + falls through to upstream) rather than drift to whatever + lemonade ships as latest. """ if not host.rocm_gfx_target: return None # Opt-out for users who want the upstream HIP build path only -- lemonade - # binaries lack approved-hash manifest entries, so their integrity gate is - # functional validation only. + # binaries are downloaded without entries in the approved-hash manifest, so + # the integrity gate is functional validation only. if os.environ.get("UNSLOTH_DISABLE_LEMONADE_ROCM", "").strip().lower() in ( "1", "true", @@ -3822,7 +3920,7 @@ def resolve_lemonade_rocm_choice( return None asset_url = assets[asset_name] if not asset_url: - # release_asset_map defaults to "" when an asset row lacks + # release_asset_map defaults to "" when an asset row is missing # browser_download_url; skip cleanly instead of letting # download_file("") raise a less obvious error downstream. log( @@ -3831,9 +3929,9 @@ def resolve_lemonade_rocm_choice( ) return None # Defence in depth: lemonade browser_download_url should be on github.com - # or githubusercontent.com. A compromised GitHub API response redirecting - # to an attacker host would otherwise be honoured silently (lemonade - # assets are not in the approved-hash manifest). + # or githubusercontent.com. A compromised GitHub API response that + # redirects to an attacker-chosen host would otherwise be honoured + # silently (lemonade assets are not in the approved-hash manifest). if not _is_trusted_github_release_url(asset_url, LEMONADE_ROCM_REPO): log( f"{LEMONADE_ROCM_REPO}@{release_tag} asset {asset_name!r} points " @@ -3841,9 +3939,9 @@ def resolve_lemonade_rocm_choice( "lemonade prebuilt" ) return None - # Note: lemonade tags Linux assets "ubuntu" but the binary is a generic - # glibc build that runs on any distro (Arch, Fedora, ...), so this attempt - # is selected for all Linux ROCm hosts, not just Ubuntu. + # Note: lemonade tags Linux assets with "ubuntu" but the binary is a + # generic glibc build that runs on any distro (Arch, Fedora, ...), so + # this attempt is selected for all Linux ROCm hosts, not just Ubuntu. log( f"AMD GPU {host.rocm_gfx_target!r} ({gfx_family}) -- " f"trying lemonade-sdk ROCm prebuilt {asset_name} " @@ -3866,30 +3964,40 @@ def resolve_lemonade_rocm_choice( ) -def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice: +def resolve_upstream_asset_choice( + host: HostInfo, + llama_tag: str, + lemonade_tag: "str | None" = None, +) -> AssetChoice: + # lemonade_tag: tag for the lemonade lookup only. The release scan pins + # llama_tag to per-release upstream tags (b9518, ...) that lemonade's own + # tag series (b1292, ...) never contains, so pinning lemonade to them 404s + # on every scanned release. Scan callers pass the original request + # (normally "latest") here; upstream asset names keep the pinned tag. upstream_assets = github_release_assets(UPSTREAM_REPO, llama_tag) if host.is_linux and host.is_x86_64: - # AMD ROCm: try upstream ROCm prebuilt first, then a source build. The - # source build (via setup.sh) compiles with -DGGML_HIP=ON and - # auto-detects the exact GPU target via rocminfo, more reliable for - # consumer GPUs (e.g. gfx1151) that may not be in the prebuilt. + # AMD ROCm: try upstream ROCm prebuilt first, then fall back to source build. + # Source build (via setup.sh) compiles with -DGGML_HIP=ON and auto-detects + # the exact GPU target via rocminfo, which is more reliable for consumer + # GPUs (e.g. gfx1151) that may not be in the prebuilt. if host.has_rocm and not host.has_usable_nvidia: - # Try lemonade-sdk per-GPU prebuilt first: built against specific - # gfx targets and bundle all required ROCm runtime libs. + # Try lemonade-sdk per-GPU prebuilt first: these are built against + # specific gfx targets and bundle all required ROCm runtime libs. lemonade_choice = resolve_lemonade_rocm_choice( - host, "ubuntu", "linux-rocm", llama_tag = llama_tag + host, "ubuntu", "linux-rocm", llama_tag = lemonade_tag or llama_tag ) if lemonade_choice is not None: return lemonade_choice - # Fall back to the upstream combined ROCm tarball. Scan for any - # rocm- prebuilt; when the host ROCm version is known, - # pick the newest candidate whose major.minor is <= host version - # -- otherwise a ROCm 6.4 host downloads the rocm-7.2 tarball, - # fails preflight, and source-builds even though a 6.4 prebuilt - # exists. If none is compatible (host older than every published - # prebuilt), fall back to the numerically newest so we try - # something. + # Fall back to upstream combined ROCm tarball. + # Scan upstream assets for any rocm- prebuilt. When the + # host ROCm runtime version is known, pick the newest candidate + # whose major.minor is <= host version -- otherwise a ROCm 6.4 + # host would download the rocm-7.2 tarball, fail preflight, and + # fall back to a source build even though a compatible 6.4 + # prebuilt exists. If no compatible candidate matches (e.g. host + # runtime is older than every published prebuilt), fall back to + # the numerically newest so we at least try something. _rocm_pattern = re.compile( rf"llama-{re.escape(llama_tag)}-bin-ubuntu-rocm-([0-9]+(?:\.[0-9]+)*)-x64\.tar\.gz" ) @@ -3908,10 +4016,10 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice item for item in rocm_candidates if item[0][:2] <= _host_rocm_version ] if rocm_candidates and not _compatible: - # Fall back to the newest candidate so we don't force a source - # build when the host runtime is older than every published - # prebuilt: preflight still catches a true incompatibility and - # triggers a fallback. + # Fall back to the newest candidate so a source build is + # not forced when the host runtime is older than every + # published prebuilt: preflight will still catch a true + # incompatibility and trigger a fallback. _compatible = rocm_candidates[:1] if _compatible: rocm_name = _compatible[0][1] @@ -3934,7 +4042,7 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice source_label = "upstream", install_kind = "linux-rocm", ) - # No ROCm prebuilt available -- fall back to a source build + # No ROCm prebuilt available -- fall back to source build raise PrebuiltFallback( "AMD ROCm detected but no upstream ROCm prebuilt found; " "falling back to source build with HIP support" @@ -3962,7 +4070,7 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice # AMD ROCm on Windows: try lemonade per-GPU prebuilt first, then upstream HIP if host.has_rocm: lemonade_choice = resolve_lemonade_rocm_choice( - host, "windows", "windows-hip", llama_tag = llama_tag + host, "windows", "windows-hip", llama_tag = lemonade_tag or llama_tag ) if lemonade_choice is not None: return lemonade_choice @@ -4021,12 +4129,16 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice raise PrebuiltFallback(f"no prebuilt policy exists for {host.system} {host.machine}") -def resolve_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice: +def resolve_asset_choice( + host: HostInfo, + llama_tag: str, + lemonade_tag: "str | None" = None, +) -> AssetChoice: if host.is_linux and host.is_x86_64 and host.has_usable_nvidia: raise PrebuiltFallback( "Linux CUDA installs require a compatible published bundle; upstream fallback is not available" ) - return resolve_upstream_asset_choice(host, llama_tag) + return resolve_upstream_asset_choice(host, llama_tag, lemonade_tag = lemonade_tag) def resolve_release_asset_choice( @@ -4034,6 +4146,7 @@ def resolve_release_asset_choice( llama_tag: str, release: PublishedReleaseBundle, checksums: ApprovedReleaseChecksums, + requested_tag: "str | None" = None, ) -> list[AssetChoice]: if host.is_windows and host.is_x86_64 and host.has_usable_nvidia: torch_preference = detect_torch_cuda_runtime_preference(host) @@ -4064,13 +4177,15 @@ def resolve_release_asset_choice( published_choice: AssetChoice | None = None if host.is_windows and host.is_x86_64: - # AMD Windows hosts prefer a hash-approved published Windows HIP - # bundle when one exists, otherwise fall through to - # resolve_asset_choice() so the upstream HIP prebuilt is tried before - # the CPU fallback. Hard-pinning the published windows-cpu bundle here - # would make the HIP path unreachable. + # AMD Windows hosts prefer the fork's per-gfx windows-rocm bundle when one + # covers the GPU; otherwise fall through to resolve_asset_choice(). Note + # that on the fork repo the upstream win-hip archive has no approved hash, + # so apply_approved_hashes drops it and an uncovered gfx lands on a HIP + # source build (auto-detecting its exact gfx) rather than an upstream + # prebuilt. We still avoid hard-pinning windows-cpu here so a CPU bundle + # never shadows that ROCm path. if host.has_rocm: - published_choice = published_asset_choice_for_kind(release, "windows-hip") + published_choice = published_rocm_choice_for_host(release, host, "windows-rocm") else: published_choice = published_asset_choice_for_kind(release, "windows-cpu") elif host.is_macos and host.is_arm64: @@ -4087,7 +4202,10 @@ def resolve_release_asset_choice( f"{release.repo}@{release.release_tag} {published_choice.name} ({exc})" ) - return apply_approved_hashes([resolve_asset_choice(host, llama_tag)], checksums) + return apply_approved_hashes( + [resolve_asset_choice(host, llama_tag, lemonade_tag = requested_tag)], + checksums, + ) def extract_archive(archive_path: Path, destination: Path) -> None: @@ -4472,7 +4590,13 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: # libraries between b9279 and b9283) without us re-enumerating # every new file. Studio only invokes llama-server and llama-quantize; # other CLIs upstream ships (llama-cli, llama-bench, ...) are skipped. - if choice.install_kind in {"linux-cpu", "linux-cuda", "linux-rocm", "linux-arm64"}: + if choice.install_kind in { + "linux-cpu", + "linux-cuda", + "linux-arm64-cuda", + "linux-rocm", + "linux-arm64", + }: return ["llama-server", "llama-quantize", "lib*.so*"] if choice.install_kind in {"macos-arm64", "macos-x64"}: return ["llama-server", "llama-quantize", "lib*.dylib"] @@ -4480,6 +4604,7 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: "windows-cpu", "windows-cuda", "windows-hip", + "windows-rocm", "windows-arm64", }: return ["llama-server.exe", "llama-quantize.exe", "*.dll"] @@ -4494,10 +4619,7 @@ def runtime_subdirs_for_choice(choice: AssetChoice) -> list[str]: (hipblaslt/library// and rocblas/library//) to sit next to their shared libraries at runtime. These trees are multi-level and cannot be handled by copy_globs (filename-only matching, flat copy).""" - if choice.source_label == "lemonade" and choice.install_kind in { - "linux-rocm", - "windows-hip", - }: + if choice.install_kind in {"linux-rocm", "windows-rocm", "windows-hip"}: return ["hipblaslt", "rocblas"] return [] @@ -5494,9 +5616,11 @@ def validate_server( # pass one (keeps backwards compatibility with older call sites). _gpu_kinds = { "linux-cuda", + "linux-arm64-cuda", "linux-rocm", "windows-cuda", "windows-hip", + "windows-rocm", "macos-arm64", } if install_kind is not None: @@ -5815,7 +5939,7 @@ def selected_source_archive_metadata( def resolve_install_attempts( llama_tag: str, host: HostInfo, published_repo: str, published_release_tag: str ) -> tuple[str, str, list[AssetChoice], ApprovedReleaseChecksums]: - requested_tag, plans = resolve_install_release_plans( + requested_tag, plans = _fork_manifest_release_plans( llama_tag, host, published_repo, @@ -5827,7 +5951,50 @@ def resolve_install_attempts( return requested_tag, plan.llama_tag, plan.attempts, plan.approved_checksums -def resolve_install_release_plans( +def _linux_published_attempts( + host: HostInfo, bundle: PublishedReleaseBundle, requested_tag: str +) -> list[AssetChoice]: + """Build the install attempts for a fork Linux host from a manifest-described + bundle: CUDA (with a CPU fallback), per-gfx ROCm (with a lemonade fallback), + or CPU. Same selection the upstream filename path used, just sourced from the + manifest instead of reconstructed from asset names.""" + attempts: list[AssetChoice] = [] + if host.has_usable_nvidia: + # Prefer the cudart major Studio loads at runtime (torch's bundled + # libcudart), not the newest detected on disk. Without this a stray + # cuda13 runtime outranks the torch cuda12 the binary links against. + torch_preference = detect_torch_cuda_runtime_preference(host) + selection = linux_cuda_choice_from_release( + host, + bundle, + preferred_runtime_line = torch_preference.runtime_line, + selection_preamble = torch_preference.selection_log, + ) + if selection is not None: + attempts.extend(selection.attempts) + if host.has_rocm and not host.has_usable_nvidia: + # Prefer the fork's own per-gfx ROCm bundle (hash-approved, ships the + # full ROCm runtime) and fall back to the external lemonade prebuilt. + # Do NOT append the CPU asset for ROCm-only hosts: if lemonade fails + # validation we want validate_prebuilt_attempts to raise PrebuiltFallback + # so the caller triggers the HIP source build, not silently install a + # CPU-only binary. + published_rocm = published_rocm_choice_for_host(bundle, host, "linux-rocm") + if published_rocm is not None: + attempts.append(published_rocm) + lemonade_choice = resolve_lemonade_rocm_choice( + host, "ubuntu", "linux-rocm", llama_tag = requested_tag + ) + if lemonade_choice is not None: + attempts.append(lemonade_choice) + else: + cpu_choice = published_asset_choice_for_kind(bundle, "linux-cpu") + if cpu_choice is not None: + attempts.append(cpu_choice) + return attempts + + +def _fork_manifest_release_plans( llama_tag: str, host: HostInfo, published_repo: str, @@ -5835,6 +6002,10 @@ def resolve_install_release_plans( *, max_release_fallbacks: int = DEFAULT_MAX_PREBUILT_RELEASE_FALLBACKS, ) -> tuple[str, list[InstallReleasePlan]]: + """Manifest-reading branch of resolve_simple_install_release_plans, used for + the fork's bundles whose GPU/arch coverage lives in + llama-prebuilt-manifest.json rather than in the filename: arm64 CUDA, Windows + CUDA, per-gfx ROCm, and macOS. Linux x64 takes the faster filename path.""" requested_tag = normalized_requested_llama_tag(llama_tag) allow_older_release_fallback = requested_tag == "latest" and not published_release_tag release_limit = max(1, max_release_fallbacks) @@ -5854,18 +6025,22 @@ def resolve_install_release_plans( checksums = resolved_release.checksums resolved_tag = bundle.upstream_tag try: - if host.is_linux and host.is_x86_64 and host.has_usable_nvidia: - linux_cuda_selection = resolve_linux_cuda_choice(host, bundle) - attempts = apply_approved_hashes(linux_cuda_selection.attempts, checksums) + if host.is_linux: + linux_attempts = _linux_published_attempts(host, bundle, requested_tag) + if not linux_attempts: + raise PrebuiltFallback("no compatible Linux prebuilt asset was found") + attempts = apply_approved_hashes(linux_attempts, checksums) if not attempts: - raise PrebuiltFallback("no compatible Linux CUDA asset was found") - log_lines(linux_cuda_selection.selection_log) + raise PrebuiltFallback("no compatible Linux prebuilt asset was found") + if attempts[0].selection_log: + log_lines(attempts[0].selection_log) else: attempts = resolve_release_asset_choice( host, resolved_tag, bundle, checksums, + requested_tag = requested_tag, ) if not attempts: raise PrebuiltFallback("no compatible prebuilt asset was found") @@ -6021,7 +6196,7 @@ def runtime_payload_health_groups(choice: AssetChoice) -> list[list[str]]: ["libggml-cpu*.so*"], ["libmtmd.so*"], ] - if choice.install_kind == "linux-cuda": + if choice.install_kind in {"linux-cuda", "linux-arm64-cuda"}: return [ ["libllama-common.so*"], ["libllama.so*"], @@ -6064,7 +6239,7 @@ def runtime_payload_health_groups(choice: AssetChoice) -> list[list[str]]: groups.append(["cublas64_*.dll"]) groups.append(["cublasLt64_*.dll"]) return groups - if choice.install_kind == "windows-hip": + if choice.install_kind in {"windows-hip", "windows-rocm"}: return [["llama.dll"], ["*hip*.dll"]] return [] @@ -6333,7 +6508,6 @@ def install_prebuilt( published_repo: str, published_release_tag: str, *, - simple_policy: bool = False, override_has_rocm: bool = False, override_rocm_gfx: str | None = None, force_cpu: bool = False, @@ -6356,20 +6530,14 @@ def install_prebuilt( log( f"no existing llama.cpp install detected at {install_dir}; performing fresh prebuilt install" ) - if simple_policy: - requested_tag, release_plans = resolve_simple_install_release_plans( - llama_tag, - host, - published_repo, - published_release_tag, - ) - else: - requested_tag, release_plans = resolve_install_release_plans( - llama_tag, - host, - published_repo, - published_release_tag, - ) + # Single resolver: linux-x64 takes the fast filename path internally, + # every other fork host reads the manifest. + requested_tag, release_plans = resolve_simple_install_release_plans( + llama_tag, + host, + published_repo, + published_release_tag, + ) if release_plans and existing_install_matches_plan(install_dir, host, release_plans[0]): current = release_plans[0] log( @@ -6469,11 +6637,6 @@ def parse_args() -> argparse.Namespace: "until a usable published llama.cpp release bundle is found." ), ) - parser.add_argument( - "--simple-policy", - action = "store_true", - help = "Use the simplified platform-specific prebuilt selection policy.", - ) parser.add_argument( "--has-rocm", action = "store_true", @@ -6620,7 +6783,6 @@ def main() -> int: llama_tag = args.llama_tag, published_repo = args.published_repo, published_release_tag = args.published_release_tag or "", - simple_policy = args.simple_policy, override_has_rocm = args.has_rocm, override_rocm_gfx = args.rocm_gfx, force_cpu = args.cpu_fallback, diff --git a/studio/setup.ps1 b/studio/setup.ps1 index c4ee08a15f..e707cfda7a 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -2556,7 +2556,9 @@ $LlamaCppDir = Join-Path $UnslothHome "llama.cpp" $NeedLlamaSourceBuild = $false $SkipPrebuiltInstall = $false $RequestedLlamaTag = if ($env:UNSLOTH_LLAMA_TAG) { $env:UNSLOTH_LLAMA_TAG } else { $DefaultLlamaTag } -$HelperReleaseRepo = "ggml-org/llama.cpp" +# GPU Windows (CUDA / ROCm) installs the fork's app-* prebuilts; CPU-only stays +# on ggml-org (the fork ships no windows-cpu bundle). Mirrors setup.sh's routing. +$HelperReleaseRepo = if ($HasNvidiaSmi -or $HasROCm) { "unslothai/llama.cpp" } else { "ggml-org/llama.cpp" } $LlamaPr = if ($env:UNSLOTH_LLAMA_PR) { $env:UNSLOTH_LLAMA_PR.Trim() } else { "" } $LlamaPrForce = if ($env:UNSLOTH_LLAMA_PR_FORCE) { $env:UNSLOTH_LLAMA_PR_FORCE.Trim() } else { $DefaultLlamaPrForce } @@ -2655,20 +2657,25 @@ if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") { if (Test-Path -LiteralPath $LlamaCppDir) { substep "Existing llama.cpp install detected -- validating staged prebuilt update before replacement" # If the existing install is the wrong kind (e.g. windows-cpu on a ROCm - # machine that should have windows-hip), remove it so the installer is + # machine that should have windows-rocm), remove it so the installer is # forced to download the correct variant rather than skipping on tag match. $existingMetaPath = Join-Path $LlamaCppDir "UNSLOTH_PREBUILT_INFO.json" if (Test-Path $existingMetaPath) { try { $existingMeta = Get-Content $existingMetaPath -Raw | ConvertFrom-Json $existingKind = $existingMeta.install_kind - # A name-inferred gfx arch (Adrenalin-only, no confirmed runtime) - # still wants the GPU (windows-hip) build -- the lemonade prebuilt - # bundles its own runtime. Treat a known arch as ROCm-capable here, - # mirroring the --rocm-gfx forward below. - $expectedKind = if ($HasROCm -or $script:ROCmGfxArch) { "windows-hip" } elseif ($HasNvidiaSmi) { "windows-cuda" } else { "windows-cpu" } - if ($existingKind -and $existingKind -ne $expectedKind) { - substep "Removing mismatched llama.cpp install (found '$existingKind', need '$expectedKind')..." + # A ROCm host may legitimately carry the fork's windows-rocm bundle + # or the upstream windows-hip fallback, so accept either and never + # treat a valid ROCm install as mismatched. A name-inferred gfx + # arch (Adrenalin-only, no confirmed runtime) still counts as + # ROCm-capable -- the lemonade prebuilt bundles its own runtime, + # mirroring the --rocm-gfx forward below. NOTE: this block is + # currently inert -- write_prebuilt_metadata does not persist an + # install_kind key, so $existingKind is always null. If that changes, + # add the remaining host kinds (e.g. windows-arm64) before relying on it. + $expectedKinds = if ($HasROCm -or $script:ROCmGfxArch) { @("windows-rocm", "windows-hip") } elseif ($HasNvidiaSmi) { @("windows-cuda") } else { @("windows-cpu") } + if ($existingKind -and ($existingKind -notin $expectedKinds)) { + substep "Removing mismatched llama.cpp install (found '$existingKind', need one of: $($expectedKinds -join ', '))..." Remove-Item -Recurse -Force -LiteralPath $LlamaCppDir -ErrorAction SilentlyContinue } } catch { @@ -2687,8 +2694,7 @@ if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") { "$PSScriptRoot\install_llama_prebuilt.py", "--install-dir", $LlamaCppDir, "--llama-tag", $RequestedLlamaTag, - "--published-repo", $HelperReleaseRepo, - "--simple-policy" + "--published-repo", $HelperReleaseRepo ) if ($HasROCm) { $prebuiltArgs += "--has-rocm" diff --git a/studio/setup.sh b/studio/setup.sh index 3e9c9ef3d6..66c03da391 100755 --- a/studio/setup.sh +++ b/studio/setup.sh @@ -812,6 +812,7 @@ if ! command -v rocminfo >/dev/null 2>&1 && [ -x /opt/rocm/bin/rocminfo ]; then PATH="$PATH:/opt/rocm/bin" fi _setup_amd_detected=false +_setup_nvidia_usable=false _setup_gfx_all="" _setup_mkt="" if command -v rocminfo >/dev/null 2>&1 && \ @@ -832,6 +833,7 @@ fi if command -v nvidia-smi >/dev/null 2>&1 && \ nvidia-smi -L 2>/dev/null | awk '/^GPU[[:space:]]+[0-9]+:/{found=1} END{exit !found}'; then + _setup_nvidia_usable=true step "gpu" "NVIDIA GPU detected" elif [ "$_setup_amd_detected" = true ]; then _setup_vis="${HIP_VISIBLE_DEVICES:-${ROCR_VISIBLE_DEVICES:-}}" @@ -910,34 +912,46 @@ _HOST_SYSTEM="$(uname -s 2>/dev/null || true)" _HOST_MACHINE="$(uname -m 2>/dev/null || true)" # Pick the release repo install_llama_prebuilt.py plans against. -# unslothai/llama.cpp ships only Linux CUDA bundles, so CPU-only Linux -# x86_64 routes to ggml-org for bin-ubuntu-x64.tar.gz. Anything with a -# GPU tool installed stays on unslothai (CUDA bundle / ROCm source build). +# The fork ships CUDA (Linux x64/arm64, Windows), ROCm (Linux/Windows) and +# macOS bundles. Only the plain CPU/Vulkan bundles still come from ggml-org, so +# CPU-only Linux (x86_64 and arm64) routes there; GPU Linux, Windows and macOS +# use unslothai. _LINUX_HAS_GPU=false -for _GPU_TOOL in nvidia-smi rocminfo amd-smi hipconfig hipinfo; do - if command -v "$_GPU_TOOL" >/dev/null 2>&1; then - _LINUX_HAS_GPU=true - break - fi -done +# Route to the fork only for a usable GPU. NVIDIA counts only when a device is +# actually enumerated (_setup_nvidia_usable, from the nvidia-smi -L probe above) +# AND not hidden via CUDA_VISIBLE_DEVICES=-1 -- mirroring install_llama_prebuilt.py's +# has_usable_nvidia. Mere nvidia-smi presence (CPU-only CUDA-toolkit containers, +# broken drivers) or a hidden GPU therefore takes the ggml-org CPU prebuilt +# instead of a slow source build. AMD is deliberately left on tooling presence, +# not usability: an unusable NVIDIA host has a good CPU prebuilt to fall back to, +# whereas tightening AMD would regress ROCm hosts exposing only hipconfig/hipinfo +# into an unnecessary CPU build. +if [ "$_setup_nvidia_usable" = true ] && [ "${CUDA_VISIBLE_DEVICES:-}" != "-1" ]; then + _LINUX_HAS_GPU=true +else + for _GPU_TOOL in rocminfo amd-smi hipconfig hipinfo; do + if command -v "$_GPU_TOOL" >/dev/null 2>&1; then + _LINUX_HAS_GPU=true + break + fi + done +fi -if [ "$_HOST_SYSTEM" = "Darwin" ]; then - _HELPER_RELEASE_REPO="ggml-org/llama.cpp" -elif [ "$_HOST_SYSTEM" = "Linux" ] \ +if [ "$_HOST_SYSTEM" = "Linux" ] \ && [ "$_HOST_MACHINE" = "x86_64" ] \ && [ "$_LINUX_HAS_GPU" = false ]; then _HELPER_RELEASE_REPO="ggml-org/llama.cpp" elif [ "$_HOST_SYSTEM" = "Linux" ] \ && { [ "$_HOST_MACHINE" = "aarch64" ] || [ "$_HOST_MACHINE" = "arm64" ]; } \ && [ "$_LINUX_HAS_GPU" = false ]; then - # Linux ARM64 (Ampere Altra, Raspberry Pi 5, GitHub `ubuntu-24.04-arm`, - # CPU-only Jetson rescue mode, ...). unslothai/llama.cpp only ships - # the Linux CUDA bundles, so without this branch the prebuilt - # resolver returns 0 attempts on every release and the installer - # falls all the way back to a source build. Upstream ggml-org ships + # CPU-only Linux ARM64 (Ampere Altra, Raspberry Pi 5, GitHub + # `ubuntu-24.04-arm`, CPU-only Jetson rescue mode, ...). The fork ships no + # arm64 CPU bundle, so without this branch the prebuilt resolver returns 0 + # attempts and the installer falls back to a source build. ggml-org ships # llama-bNNNN-bin-ubuntu-arm64.tar.gz from at least b9072 onward. _HELPER_RELEASE_REPO="ggml-org/llama.cpp" else + # GPU Linux (x64 CUDA/ROCm, arm64 CUDA), Windows (CUDA/ROCm), and macOS. _HELPER_RELEASE_REPO="unslothai/llama.cpp" fi unset _GPU_TOOL @@ -1000,7 +1014,6 @@ else --install-dir "$LLAMA_CPP_DIR" --llama-tag "$_REQUESTED_LLAMA_TAG" --published-repo "$_HELPER_RELEASE_REPO" - --simple-policy ) if [ -n "${UNSLOTH_LLAMA_RELEASE_TAG:-}" ]; then _PREBUILT_CMD+=(--published-release-tag "$UNSLOTH_LLAMA_RELEASE_TAG") @@ -1555,7 +1568,6 @@ if [ "$_LLAMA_CPP_DEGRADED" = true ] \ --install-dir "$LLAMA_CPP_DIR" --llama-tag "$_REQUESTED_LLAMA_TAG" --published-repo "ggml-org/llama.cpp" - --simple-policy --cpu-fallback ) # Trust the installer's exit code: it validates the server before exiting 0, diff --git a/tests/studio/install/test_install_llama_prebuilt_logic.py b/tests/studio/install/test_install_llama_prebuilt_logic.py index de3808469c..057612d27b 100644 --- a/tests/studio/install/test_install_llama_prebuilt_logic.py +++ b/tests/studio/install/test_install_llama_prebuilt_logic.py @@ -338,336 +338,6 @@ def test_validate_prebuilt_choice_creates_repo_shaped_linux_install( assert (install_dir / "BUILD_INFO.txt").exists() -def test_simple_linux_direct_release_uses_published_source_checksums_for_branch( - monkeypatch: pytest.MonkeyPatch, -): - source_commit = "25b1bc9c2f9aa0a390b968ee1ffd9ff01340a3fe" - release = { - "tag_name": "llama-prebuilt-master-3a92bc9", - "assets": [ - { - "name": "app-master-linux-x64-cuda13-newer.tar.gz", - "browser_download_url": "https://example.test/app-master-linux-x64-cuda13-newer.tar.gz", - }, - { - "name": "llama-prebuilt-sha256.json", - "browser_download_url": "https://example.test/llama-prebuilt-sha256.json", - }, - ], - } - checksums = ApprovedReleaseChecksums( - repo = "unslothai/llama.cpp", - release_tag = "llama-prebuilt-master-3a92bc9", - upstream_tag = "b9174", - source_commit = source_commit, - source_repo = "ggml-org/llama.cpp", - source_repo_url = "https://github.com/ggml-org/llama.cpp", - source_ref_kind = "branch", - requested_source_ref = "master", - resolved_source_ref = "master", - artifacts = { - "app-master-linux-x64-cuda13-newer.tar.gz": ApprovedArtifactHash( - asset_name = "app-master-linux-x64-cuda13-newer.tar.gz", - sha256 = "a" * 64, - repo = "unslothai/llama.cpp", - kind = "linux-cuda-app", - ), - INSTALL_LLAMA_PREBUILT.exact_source_archive_logical_name( - source_commit - ): ApprovedArtifactHash( - asset_name = INSTALL_LLAMA_PREBUILT.exact_source_archive_logical_name(source_commit), - sha256 = "b" * 64, - repo = "ggml-org/llama.cpp", - kind = "exact-source", - ), - }, - ) - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "load_approved_release_checksums", - lambda repo, release_tag: checksums, - ) - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "detected_linux_runtime_lines", - lambda: (["cuda13"], {"cuda13": ["/usr/local/cuda/lib64"]}), - ) - host = HostInfo( - system = "Linux", - machine = "x86_64", - is_windows = False, - is_linux = True, - is_macos = False, - is_x86_64 = True, - is_arm64 = False, - nvidia_smi = None, - driver_cuda_version = (13, 1), - compute_caps = ["100"], - visible_cuda_devices = None, - has_physical_nvidia = True, - has_usable_nvidia = True, - ) - - plan = INSTALL_LLAMA_PREBUILT.direct_linux_release_plan( - release, - host, - "unslothai/llama.cpp", - "latest", - ) - - assert plan is not None - assert plan.llama_tag == "master" - assert plan.approved_checksums.upstream_tag == "b9174" - assert plan.approved_checksums.source_commit == source_commit - assert plan.attempts[0].expected_sha256 == "a" * 64 - source_repo, source_ref, _source_archive, exact_source = ( - INSTALL_LLAMA_PREBUILT.preferred_source_archive(plan.approved_checksums, plan.llama_tag) - ) - assert source_repo == "ggml-org/llama.cpp" - assert source_ref == source_commit - assert exact_source is True - - -def test_simple_linux_direct_release_honors_torch_cudart_preference( - monkeypatch: pytest.MonkeyPatch, -): - # Regression: a Blackwell host (sm_120, driver 13.0) with BOTH cudart majors - # visible -- a stray cuda13 wheel plus torch's cuda12 -- must install the - # cuda12 build that matches the runtime torch, not the newest-major cuda13 - # build (which loads no GPU and silently falls back to CPU). - release = { - "tag_name": "b9334", - "assets": [ - { - "name": f"app-b9334-linux-x64-{profile}.tar.gz", - "browser_download_url": f"https://example.test/app-b9334-linux-x64-{profile}.tar.gz", - } - for profile in ( - "cuda12-newer", - "cuda12-portable", - "cuda13-newer", - "cuda13-portable", - ) - ], - } - # cuda13 detected first (newest-major order); both compatible with driver 13.0. - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "detected_linux_runtime_lines", - lambda: ( - ["cuda13", "cuda12"], - { - "cuda13": ["/usr/local/lib/python3.13/site-packages/nvidia/cu13/lib"], - "cuda12": ["/venv/lib/python3.13/site-packages/nvidia/cuda_runtime/lib"], - }, - ), - ) - host = HostInfo( - system = "Linux", - machine = "x86_64", - is_windows = False, - is_linux = True, - is_macos = False, - is_x86_64 = True, - is_arm64 = False, - nvidia_smi = "nvidia-smi", - driver_cuda_version = (13, 0), - compute_caps = ["120"], - visible_cuda_devices = None, - has_physical_nvidia = True, - has_usable_nvidia = True, - ) - - def first_asset_for_torch(line): - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "detect_torch_cuda_runtime_preference", - lambda h: INSTALL_LLAMA_PREBUILT.CudaRuntimePreference( - runtime_line = line, selection_log = [] - ), - ) - plan = INSTALL_LLAMA_PREBUILT.direct_linux_release_plan( - release, host, "unslothai/llama.cpp", "latest" - ) - return plan.attempts[0] - - # torch reports cuda12 (the cu128 runtime) -> install the cuda12 build. - primary = first_asset_for_torch("cuda12") - assert primary.name == "app-b9334-linux-x64-cuda12-newer.tar.gz" - assert primary.runtime_line == "cuda12" - - # torch unavailable -> unchanged newest-major fallback (documents the residual). - assert first_asset_for_torch(None).name == "app-b9334-linux-x64-cuda13-newer.tar.gz" - - -@pytest.mark.parametrize( - "mutate, expected_match", - [ - # Missing source_commit. - ( - lambda c: setattr(c, "source_commit", None) or setattr(c, "source_commit_short", None), - "exact source provenance", - ), - # source_commit present, but no exact-source archive hash. - ( - lambda c: c.artifacts.pop( - INSTALL_LLAMA_PREBUILT.exact_source_archive_logical_name(c.source_commit), - None, - ), - "exact source provenance", - ), - # source_commit + exact-source archive present, but no source_repo. - ( - lambda c: setattr(c, "source_repo", None) or setattr(c, "source_repo_url", None), - "exact source provenance", - ), - ], - ids = [ - "missing_source_commit", - "missing_exact_source_artifact", - "missing_source_repo", - ], -) -def test_simple_linux_direct_release_rejects_branch_without_exact_source_metadata( - monkeypatch: pytest.MonkeyPatch, mutate, expected_match -): - source_commit = "25b1bc9c2f9aa0a390b968ee1ffd9ff01340a3fe" - release = { - "tag_name": "llama-prebuilt-master-3a92bc9", - "assets": [ - { - "name": "app-master-linux-x64-cuda13-newer.tar.gz", - "browser_download_url": "https://example.test/app-master-linux-x64-cuda13-newer.tar.gz", - }, - { - "name": "llama-prebuilt-sha256.json", - "browser_download_url": "https://example.test/llama-prebuilt-sha256.json", - }, - ], - } - checksums = ApprovedReleaseChecksums( - repo = "unslothai/llama.cpp", - release_tag = "llama-prebuilt-master-3a92bc9", - upstream_tag = "b9174", - source_commit = source_commit, - source_repo = "ggml-org/llama.cpp", - source_repo_url = "https://github.com/ggml-org/llama.cpp", - source_ref_kind = "branch", - requested_source_ref = "master", - resolved_source_ref = "master", - artifacts = { - "app-master-linux-x64-cuda13-newer.tar.gz": ApprovedArtifactHash( - asset_name = "app-master-linux-x64-cuda13-newer.tar.gz", - sha256 = "a" * 64, - repo = "unslothai/llama.cpp", - kind = "linux-cuda-app", - ), - INSTALL_LLAMA_PREBUILT.exact_source_archive_logical_name( - source_commit - ): ApprovedArtifactHash( - asset_name = INSTALL_LLAMA_PREBUILT.exact_source_archive_logical_name(source_commit), - sha256 = "b" * 64, - repo = "ggml-org/llama.cpp", - kind = "exact-source", - ), - }, - ) - mutate(checksums) - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "load_approved_release_checksums", - lambda repo, release_tag: checksums, - ) - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "detected_linux_runtime_lines", - lambda: (["cuda13"], {"cuda13": ["/usr/local/cuda/lib64"]}), - ) - host = HostInfo( - system = "Linux", - machine = "x86_64", - is_windows = False, - is_linux = True, - is_macos = False, - is_x86_64 = True, - is_arm64 = False, - nvidia_smi = None, - driver_cuda_version = (13, 1), - compute_caps = ["100"], - visible_cuda_devices = None, - has_physical_nvidia = True, - has_usable_nvidia = True, - ) - - with pytest.raises(PrebuiltFallback, match = expected_match): - INSTALL_LLAMA_PREBUILT.direct_linux_release_plan( - release, - host, - "unslothai/llama.cpp", - "latest", - ) - - -def test_simple_linux_direct_release_keeps_legacy_b_tag_path_without_checksums( - monkeypatch: pytest.MonkeyPatch, -): - release = { - "tag_name": "b9999", - "assets": [ - { - "name": "app-b9999-linux-x64-cuda13-newer.tar.gz", - "browser_download_url": "https://example.test/app-b9999-linux-x64-cuda13-newer.tar.gz", - }, - { - "name": "llama-prebuilt-sha256.json", - "browser_download_url": "https://example.test/llama-prebuilt-sha256.json", - }, - ], - } - - def unexpected_checksum_load(repo: str, release_tag: str): - raise AssertionError("legacy b-tag direct releases should not require checksum metadata") - - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "load_approved_release_checksums", - unexpected_checksum_load, - ) - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "detected_linux_runtime_lines", - lambda: (["cuda13"], {"cuda13": ["/usr/local/cuda/lib64"]}), - ) - host = HostInfo( - system = "Linux", - machine = "x86_64", - is_windows = False, - is_linux = True, - is_macos = False, - is_x86_64 = True, - is_arm64 = False, - nvidia_smi = None, - driver_cuda_version = (13, 1), - compute_caps = ["100"], - visible_cuda_devices = None, - has_physical_nvidia = True, - has_usable_nvidia = True, - ) - - plan = INSTALL_LLAMA_PREBUILT.direct_linux_release_plan( - release, - host, - "unslothai/llama.cpp", - "latest", - ) - - assert plan is not None - assert plan.llama_tag == "b9999" - assert plan.release_tag == "b9999" - assert plan.approved_checksums.source_commit is None - assert plan.attempts[0].expected_sha256 is None - - def test_validate_prebuilt_choice_creates_repo_shaped_windows_install( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ): @@ -993,7 +663,7 @@ def test_install_prebuilt_falls_back_to_older_release_plan( monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda: host) monkeypatch.setattr( INSTALL_LLAMA_PREBUILT, - "resolve_install_release_plans", + "resolve_simple_install_release_plans", lambda llama_tag, host, published_repo, published_release_tag: ( "latest", [first_plan, second_plan], @@ -1921,7 +1591,7 @@ def test_install_prebuilt_skips_download_when_existing_install_matches( monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda: host) monkeypatch.setattr( INSTALL_LLAMA_PREBUILT, - "resolve_install_release_plans", + "resolve_simple_install_release_plans", lambda llama_tag, host, published_repo, published_release_tag: ( "latest", [plan], @@ -2011,7 +1681,7 @@ def test_install_prebuilt_does_not_skip_unhealthy_existing_install( monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda: host) monkeypatch.setattr( INSTALL_LLAMA_PREBUILT, - "resolve_install_release_plans", + "resolve_simple_install_release_plans", lambda llama_tag, host, published_repo, published_release_tag: ( "latest", [plan], @@ -2139,7 +1809,7 @@ def test_install_prebuilt_skips_when_older_release_fallback_matches_existing_ins monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda: host) monkeypatch.setattr( INSTALL_LLAMA_PREBUILT, - "resolve_install_release_plans", + "resolve_simple_install_release_plans", lambda llama_tag, host, published_repo, published_release_tag: ( "latest", [latest_plan, fallback_plan], @@ -2286,7 +1956,7 @@ def test_install_prebuilt_skips_same_release_fallback_attempt_when_installed( monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda: host) monkeypatch.setattr( INSTALL_LLAMA_PREBUILT, - "resolve_install_release_plans", + "resolve_simple_install_release_plans", lambda llama_tag, host, published_repo, published_release_tag: ( "latest", [plan], @@ -2405,7 +2075,7 @@ def test_install_prebuilt_same_tag_upstream_failure_uses_older_unsloth_release_p monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "detect_host", lambda: host) monkeypatch.setattr( INSTALL_LLAMA_PREBUILT, - "resolve_install_release_plans", + "resolve_simple_install_release_plans", lambda llama_tag, host, published_repo, published_release_tag: ( "latest", [latest_plan, older_plan], diff --git a/tests/studio/install/test_llama_pr_force_and_source.py b/tests/studio/install/test_llama_pr_force_and_source.py index 8e2a56ae39..6c780057ad 100644 --- a/tests/studio/install/test_llama_pr_force_and_source.py +++ b/tests/studio/install/test_llama_pr_force_and_source.py @@ -463,8 +463,13 @@ class TestSourcePatternsPs1: assert "$LlamaSource = $DefaultLlamaSource" in self.content def test_release_repo_override_removed(self): + # No env-based release-repo override; the repo is chosen by GPU detection + # (GPU -> fork, CPU -> ggml-org), mirroring setup.sh. assert "$HelperReleaseRepo = if ($env:UNSLOTH_LLAMA_RELEASE_REPO)" not in self.content - assert '$HelperReleaseRepo = "ggml-org/llama.cpp"' in self.content + assert ( + "$HelperReleaseRepo = if ($HasNvidiaSmi -or $HasROCm) " + '{ "unslothai/llama.cpp" } else { "ggml-org/llama.cpp" }' in self.content + ) def test_force_compile_skips_prebuilt_resolution_early(self): assert 'if ($env:UNSLOTH_LLAMA_FORCE_COMPILE -eq "1") {' in self.content diff --git a/tests/studio/install/test_pr4562_bugfixes.py b/tests/studio/install/test_pr4562_bugfixes.py index 5b1c4a44e5..9dfa4e0005 100644 --- a/tests/studio/install/test_pr4562_bugfixes.py +++ b/tests/studio/install/test_pr4562_bugfixes.py @@ -686,14 +686,26 @@ class TestSourceCodePatterns: assert "_RESOLVED_SOURCE_REF_KIND" in content assert "_RESOLVED_SOURCE_REF" in content - def test_setup_sh_prebuilt_install_uses_simple_policy_only(self): - """Shell prebuilt path should use the simplified helper install entrypoint.""" + def test_setup_sh_prebuilt_install_entrypoint(self): + """Shell prebuilt path should call the helper install entrypoint, not the + old tag-resolution / releases-latest flow.""" content = SETUP_SH.read_text() - assert "--simple-policy" in content assert "--resolve-install-tag" not in content assert "_HELPER_RELEASE_REPO}/releases/latest" not in content assert "ggml-org/llama.cpp/releases/latest" not in content + def test_setup_sh_routes_to_fork_only_on_usable_gpu(self): + """Linux fork-vs-ggml routing must gate NVIDIA on actual GPU usability, + not mere nvidia-smi presence, so CPU-only / hidden-GPU hosts (e.g. + CUDA_VISIBLE_DEVICES=-1) get the ggml CPU prebuilt instead of a source + build. Guards against a silent revert to the old presence-only loop.""" + content = SETUP_SH.read_text() + assert '[ "$_setup_nvidia_usable" = true ]' in content + assert "CUDA_VISIBLE_DEVICES" in content + # nvidia-smi must NOT be back in the bare presence loop. + assert "for _GPU_TOOL in nvidia-smi" not in content + assert "for _GPU_TOOL in rocminfo amd-smi hipconfig hipinfo" in content + def test_setup_sh_reports_installed_prebuilt_release(self): """Shell wrapper should report the installed prebuilt release from metadata.""" content = SETUP_SH.read_text() @@ -832,10 +844,10 @@ class TestSourceCodePatterns: if "LlamaCppDir" in context: pytest.fail(f"Found 'git pull' in llama.cpp build section at line {i+1}") - def test_setup_ps1_prebuilt_install_uses_simple_policy_only(self): - """PS1 prebuilt path should use the simplified helper install entrypoint.""" + def test_setup_ps1_prebuilt_install_entrypoint(self): + """PS1 prebuilt path should call the helper install entrypoint, not the + old tag-resolution / releases-latest flow.""" content = SETUP_PS1.read_text() - assert '"--simple-policy"' in content assert "--resolve-install-tag" not in content assert "$HelperReleaseRepo/releases/latest" not in content assert "ggml-org/llama.cpp/releases/latest" not in content diff --git a/tests/studio/install/test_selection_logic.py b/tests/studio/install/test_selection_logic.py index dfa2378ef5..a4fc09a183 100644 --- a/tests/studio/install/test_selection_logic.py +++ b/tests/studio/install/test_selection_logic.py @@ -54,12 +54,11 @@ compatible_windows_runtime_lines = INSTALL_LLAMA_PREBUILT.compatible_windows_run runtime_line_from_cuda_version = INSTALL_LLAMA_PREBUILT.runtime_line_from_cuda_version apply_approved_hashes = INSTALL_LLAMA_PREBUILT.apply_approved_hashes linux_cuda_choice_from_release = INSTALL_LLAMA_PREBUILT.linux_cuda_choice_from_release -parse_direct_linux_release_bundle = INSTALL_LLAMA_PREBUILT.parse_direct_linux_release_bundle windows_cuda_attempts = INSTALL_LLAMA_PREBUILT.windows_cuda_attempts resolve_upstream_asset_choice = INSTALL_LLAMA_PREBUILT.resolve_upstream_asset_choice resolve_requested_install_tag = INSTALL_LLAMA_PREBUILT.resolve_requested_install_tag resolve_install_attempts = INSTALL_LLAMA_PREBUILT.resolve_install_attempts -resolve_install_release_plans = INSTALL_LLAMA_PREBUILT.resolve_install_release_plans +_fork_manifest_release_plans = INSTALL_LLAMA_PREBUILT._fork_manifest_release_plans resolve_published_release = INSTALL_LLAMA_PREBUILT.resolve_published_release resolve_source_build_plan = INSTALL_LLAMA_PREBUILT.resolve_source_build_plan validated_checksums_for_bundle = INSTALL_LLAMA_PREBUILT.validated_checksums_for_bundle @@ -114,7 +113,9 @@ def load_studio_run_module(monkeypatch): return module +# --------------------------------------------------------------------------- # Helper factories +# --------------------------------------------------------------------------- def make_host(**overrides): @@ -272,7 +273,9 @@ def mock_windows_runtime(monkeypatch, lines): ) +# =========================================================================== # Studio run.py localhost warning +# =========================================================================== class TestStudioLocalhostIpv6Warning: @@ -453,7 +456,9 @@ class TestStudioLocalhostIpv6Warning: assert calls["stop_hint"] == 1 +# =========================================================================== # A. normalize_compute_cap +# =========================================================================== class TestNormalizeComputeCap: @@ -485,7 +490,9 @@ class TestNormalizeComputeCap: assert normalize_compute_cap("9.0") == "90" +# =========================================================================== # B. normalize_compute_caps +# =========================================================================== class TestNormalizeComputeCaps: @@ -502,7 +509,9 @@ class TestNormalizeComputeCaps: assert normalize_compute_caps([]) == [] +# =========================================================================== # C. parse_cuda_visible_devices +# =========================================================================== class TestParseCudaVisibleDevices: @@ -525,7 +534,9 @@ class TestParseCudaVisibleDevices: assert parse_cuda_visible_devices(" 0 , 1 ") == ["0", "1"] +# =========================================================================== # D. supports_explicit_visible_device_matching +# =========================================================================== class TestSupportsExplicitVisibleDeviceMatching: @@ -545,7 +556,9 @@ class TestSupportsExplicitVisibleDeviceMatching: assert supports_explicit_visible_device_matching(["0", "MIG-device"]) is False +# =========================================================================== # E. select_visible_gpu_rows +# =========================================================================== class TestSelectVisibleGpuRows: @@ -578,7 +591,9 @@ class TestSelectVisibleGpuRows: assert result == [] +# =========================================================================== # F. compatible_linux_runtime_lines +# =========================================================================== class TestCompatibleLinuxRuntimeLines: @@ -604,38 +619,9 @@ class TestCompatibleLinuxRuntimeLines: assert compatible_linux_runtime_lines(host) == ["cuda14", "cuda13", "cuda12"] -class TestParseDirectLinuxReleaseBundle: - def _release(self, *targets): - names = [f"app-bTEST-linux-x64-{t}.tar.gz" for t in targets] - return { - "tag_name": "bTEST", - "assets": [{"name": n, "browser_download_url": "https://x/" + n} for n in names], - } - - def _cuda_artifact(self, bundle): - return [a for a in bundle.artifacts if a.install_kind == "linux-cuda"][0] - - def test_parses_known_cuda13_bundle(self): - bundle = parse_direct_linux_release_bundle( - "unslothai/llama.cpp", self._release("cuda13-newer") - ) - assert bundle is not None - assert self._cuda_artifact(bundle).runtime_line == "cuda13" - - def test_parses_future_cuda_major_with_forward_profile(self): - # A future major name parses and inherits the newest known major's - # coverage for the same class as a forward default. - bundle = parse_direct_linux_release_bundle( - "unslothai/llama.cpp", self._release("cuda14-newer") - ) - assert bundle is not None - art = self._cuda_artifact(bundle) - assert art.runtime_line == "cuda14" - assert art.coverage_class == "newer" - assert art.max_sm == 120 # inherited from cuda13-newer - - +# =========================================================================== # G. pick_windows_cuda_runtime + compatible_windows_runtime_lines +# =========================================================================== class TestPickWindowsCudaRuntime: @@ -669,6 +655,14 @@ class TestCompatibleWindowsRuntimeLines: host = make_host(driver_cuda_version = (12, 4)) assert compatible_windows_runtime_lines(host) == ["cuda12"] + @pytest.mark.parametrize("minor", [0, 1, 2, 3]) + def test_cuda12_runs_on_any_12_x_driver(self, minor): + # cuda12 app bundles are toolkit-12.8 builds with bundled runtime; CUDA + # minor-version compatibility runs them on any 12.x driver, same as Linux. + # Previously Windows wrongly gated cuda12 below a 12.4 driver. + host = make_host(driver_cuda_version = (12, minor)) + assert compatible_windows_runtime_lines(host) == ["cuda12"] + def test_driver_13_1(self): host = make_host(driver_cuda_version = (13, 1)) assert compatible_windows_runtime_lines(host) == ["cuda13", "cuda12"] @@ -682,7 +676,9 @@ class TestCompatibleWindowsRuntimeLines: assert compatible_windows_runtime_lines(host) == ["cuda14", "cuda13", "cuda12"] +# =========================================================================== # H. runtime_line_from_cuda_version +# =========================================================================== class TestRuntimeLineFromCudaVersion: @@ -702,7 +698,9 @@ class TestRuntimeLineFromCudaVersion: assert runtime_line_from_cuda_version("") is None +# =========================================================================== # I. apply_approved_hashes +# =========================================================================== class TestApplyApprovedHashes: @@ -816,7 +814,9 @@ class TestApplyApprovedHashes: apply_approved_hashes([], checksums) +# =========================================================================== # J. published release resolution +# =========================================================================== class TestPublishedReleaseResolution: @@ -948,6 +948,7 @@ class TestPublishedReleaseResolution: [], release_tag = release_tag, upstream_tag = "b9000", + source_repo = "example/custom-llama.cpp", source_commit = commit, ), ) @@ -1149,8 +1150,55 @@ class TestValidatedChecksumsForBundle: with pytest.raises(PrebuiltFallback, match = "manifest checksum"): validated_checksums_for_bundle("unslothai/llama.cpp", bundle) + def test_rejects_exact_source_without_repo(self, monkeypatch): + # An exact source archive with no source repo to clone from would let + # preferred_source_archive silently fall back to upstream source at the + # tag, so validation must fail closed (clean source build instead). + bundle = make_release([], release_tag = "r1", upstream_tag = "b8508") + checksums = make_checksums_with_source( + [], release_tag = "r1", upstream_tag = "b8508", source_commit = "a" * 40 + ) # exact source archive, but no source_repo + monkeypatch.setattr( + INSTALL_LLAMA_PREBUILT, + "load_approved_release_checksums", + lambda repo, release_tag: checksums, + ) + with pytest.raises(PrebuiltFallback, match = "exact source archive"): + validated_checksums_for_bundle("unslothai/llama.cpp", bundle) + + def test_accepts_exact_source_when_only_bundle_has_repo(self, monkeypatch): + # The source repo can live only in the manifest bundle, not the checksum + # payload. source_build_plan_for_release coalesces checksums-or-bundle, so + # validation must accept the bundle's repo rather than failing closed. + bundle = make_release( + [], + release_tag = "r1", + upstream_tag = "b8508", + source_repo = "ggml-org/llama.cpp", + source_repo_url = "https://github.com/ggml-org/llama.cpp", + ) + checksums = make_checksums_with_source( + [], release_tag = "r1", upstream_tag = "b8508", source_commit = "a" * 40 + ) # exact source archive, repo only on the bundle + monkeypatch.setattr( + INSTALL_LLAMA_PREBUILT, + "load_approved_release_checksums", + lambda repo, release_tag: checksums, + ) + + assert validated_checksums_for_bundle("unslothai/llama.cpp", bundle) is checksums + plan = INSTALL_LLAMA_PREBUILT.source_build_plan_for_release( + INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease(bundle = bundle, checksums = checksums) + ) + assert plan.source_url == "https://github.com/ggml-org/llama.cpp" + assert plan.source_ref_kind == "commit" + assert plan.source_ref == "a" * 40 + + +# =========================================================================== # K. linux_cuda_choice_from_release -- core selection +# =========================================================================== class TestLinuxCudaChoiceFromRelease: @@ -1200,6 +1248,36 @@ class TestLinuxCudaChoiceFromRelease: log_entries = result.selection_log assert any("unavailable_on_host" in entry for entry in log_entries) + def test_arm64_host_selects_linux_arm64_cuda_kind(self, monkeypatch): + # An arm64 CUDA host (DGX Spark / Grace Hopper) selects the + # linux-arm64-cuda bundle and ignores the x64 linux-cuda one. + mock_linux_runtime(monkeypatch, ["cuda13"]) + host = make_host( + machine = "aarch64", + driver_cuda_version = (13, 0), + compute_caps = ["90"], + ) + arm = make_artifact( + "app-b9457-linux-arm64-cuda13-portable.tar.gz", + install_kind = "linux-arm64-cuda", + runtime_line = "cuda13", + coverage_class = "portable", + supported_sms = ["90", "100", "120", "121"], + min_sm = 90, + max_sm = 121, + bundle_profile = "cuda13-portable", + ) + x64 = make_artifact( + "app-b9457-linux-x64-cuda13-portable.tar.gz", + install_kind = "linux-cuda", + runtime_line = "cuda13", + ) + release = make_release([arm, x64]) + result = linux_cuda_choice_from_release(host, release) + assert result is not None + assert result.primary.install_kind == "linux-arm64-cuda" + assert result.primary.name == "app-b9457-linux-arm64-cuda13-portable.tar.gz" + # --- SM matching --- def test_exact_sm_match(self, monkeypatch): @@ -1404,80 +1482,9 @@ class TestLinuxCudaChoiceFromRelease: assert result is None -def make_profile_artifact(asset_name, profile_name, **overrides): - profile = INSTALL_LLAMA_PREBUILT.DIRECT_LINUX_BUNDLE_PROFILES[profile_name] - defaults = dict( - runtime_line = profile["runtime_line"], - coverage_class = profile["coverage_class"], - supported_sms = [str(value) for value in profile["supported_sms"]], - min_sm = int(profile["min_sm"]), - max_sm = int(profile["max_sm"]), - bundle_profile = profile_name, - rank = int(profile["rank"]), - ) - defaults.update(overrides) - return make_artifact(asset_name, **defaults) - - -class TestBlackwellUltraSm103Coverage: - """sm_103 (B300 / GB300) runs on the bundled base compute_100 PTX via JIT.""" - - def test_profiles_list_sm103_wherever_sm100_is_shipped(self): - for ( - name, - profile, - ) in INSTALL_LLAMA_PREBUILT.DIRECT_LINUX_BUNDLE_PROFILES.items(): - sms = {str(value) for value in profile["supported_sms"]} - if "100" in sms: - assert "103" in sms, name - else: - assert "103" not in sms, name - - def test_b300_selects_cuda13_newer_prebuilt(self, monkeypatch): - mock_linux_runtime(monkeypatch, ["cuda13"]) - host = make_host(compute_caps = ["103"], driver_cuda_version = (13, 0)) - art = make_profile_artifact("cuda13-newer.tar.gz", "cuda13-newer") - release = make_release([art]) - result = linux_cuda_choice_from_release(host, release) - assert result is not None - assert result.primary.name == "cuda13-newer.tar.gz" - - def test_b300_selects_cuda12_newer_prebuilt(self, monkeypatch): - mock_linux_runtime(monkeypatch, ["cuda12"]) - host = make_host(compute_caps = ["103"], driver_cuda_version = (12, 8)) - art = make_profile_artifact("cuda12-newer.tar.gz", "cuda12-newer") - release = make_release([art]) - result = linux_cuda_choice_from_release(host, release) - assert result is not None - assert result.primary.name == "cuda12-newer.tar.gz" - - def test_b300_reported_as_decimal_normalizes_and_matches(self, monkeypatch): - mock_linux_runtime(monkeypatch, ["cuda13"]) - host = make_host(compute_caps = ["10.3"], driver_cuda_version = (13, 0)) - art = make_profile_artifact("cuda13-portable.tar.gz", "cuda13-portable") - release = make_release([art]) - result = linux_cuda_choice_from_release(host, release) - assert result is not None - - def test_b300_falls_back_to_portable_when_only_portable_present(self, monkeypatch): - mock_linux_runtime(monkeypatch, ["cuda13"]) - host = make_host(compute_caps = ["103"], driver_cuda_version = (13, 0)) - art = make_profile_artifact("cuda13-portable.tar.gz", "cuda13-portable") - release = make_release([art]) - result = linux_cuda_choice_from_release(host, release) - assert result is not None - assert result.primary.name == "cuda13-portable.tar.gz" - - def test_older_bundle_still_rejects_b300(self, monkeypatch): - mock_linux_runtime(monkeypatch, ["cuda13"]) - host = make_host(compute_caps = ["103"], driver_cuda_version = (13, 0)) - art = make_profile_artifact("cuda13-older.tar.gz", "cuda13-older") - release = make_release([art]) - result = linux_cuda_choice_from_release(host, release) - assert result is None - - +# =========================================================================== # L. resolve_install_attempts +# =========================================================================== class TestResolveInstallAttempts: @@ -1602,7 +1609,11 @@ class TestResolveInstallAttempts: assert attempts[0].expected_sha256 == "a" * 64 assert approved.release_tag == "llama-prebuilt-latest" - def test_linux_cpu_uses_same_tag_upstream_asset(self, monkeypatch): + def test_linux_cpu_fork_without_bundle_raises_no_upstream_fallback(self, monkeypatch): + # A CPU-only Linux host on the fork no longer falls back to the ggml-org + # CPU asset: production routes CPU-only Linux to ggml-org, never the fork. + # With no fork CPU bundle in the manifest the resolver raises rather than + # quietly reaching for an upstream asset. host = make_host( has_usable_nvidia = False, has_physical_nvidia = False, @@ -1610,7 +1621,7 @@ class TestResolveInstallAttempts: ) release = make_release([], release_tag = "llama-prebuilt-latest", upstream_tag = "b9000") checksums = make_checksums_with_source( - ["llama-b9000-bin-ubuntu-x64.tar.gz"], + [], release_tag = release.release_tag, upstream_tag = "b9000", ) @@ -1630,22 +1641,13 @@ class TestResolveInstallAttempts: monkeypatch.setattr( INSTALL_LLAMA_PREBUILT, "github_release_assets", - lambda repo, tag: { - f"llama-{tag}-bin-ubuntu-x64.tar.gz": f"https://example.com/llama-{tag}-bin-ubuntu-x64.tar.gz" - }, + lambda repo, tag: (_ for _ in ()).throw( + AssertionError("fork CPU host must not query upstream assets") + ), ) - _requested_tag, resolved_tag, attempts, _approved = resolve_install_attempts( - "latest", - host, - "unslothai/llama.cpp", - "", - ) - - assert resolved_tag == "b9000" - assert attempts[0].name == "llama-b9000-bin-ubuntu-x64.tar.gz" - assert attempts[0].source_label == "upstream" - assert attempts[0].expected_sha256 == "a" * 64 + with pytest.raises(PrebuiltFallback, match = "no compatible Linux prebuilt asset was found"): + resolve_install_attempts("latest", host, "unslothai/llama.cpp", "") def test_linux_cuda_does_not_fall_back_to_upstream_cpu(self, monkeypatch): host = make_host(system = "Linux", machine = "x86_64", compute_caps = ["86"]) @@ -1670,7 +1672,7 @@ class TestResolveInstallAttempts: ) mock_linux_runtime(monkeypatch, ["cuda12"]) - with pytest.raises(PrebuiltFallback, match = "no compatible published Linux CUDA bundle"): + with pytest.raises(PrebuiltFallback, match = "no compatible Linux prebuilt asset was found"): resolve_install_attempts("latest", host, "unslothai/llama.cpp", "") def test_windows_cpu_prefers_published_asset(self, monkeypatch): @@ -1866,37 +1868,35 @@ class TestResolveInstallAttempts: class TestResolveInstallReleasePlans: - def test_latest_collects_multiple_older_release_plans_up_to_limit(self, monkeypatch): - host = make_host( - has_usable_nvidia = False, - has_physical_nvidia = False, - nvidia_smi = None, + def _cuda_bundle(self, asset_name, release_tag, upstream_tag): + # A fork CUDA bundle that covers the default NVIDIA host (sm 86, + # cuda12 runtime), so each release yields a plan via + # linux_cuda_choice_from_release. + art = make_artifact( + asset_name, + install_kind = "linux-cuda", + runtime_line = "cuda12", + coverage_class = "portable", + supported_sms = ["75", "80", "86", "89", "90"], + min_sm = 75, + max_sm = 90, ) + return INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease( + bundle = make_release([art], release_tag = release_tag, upstream_tag = upstream_tag), + checksums = make_checksums_with_source( + [asset_name], + release_tag = release_tag, + upstream_tag = upstream_tag, + ), + ) + + def test_latest_collects_multiple_older_release_plans_up_to_limit(self, monkeypatch): + mock_linux_runtime(monkeypatch, ["cuda12"]) + host = make_host(system = "Linux", machine = "x86_64", compute_caps = ["86"]) releases = [ - INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease( - bundle = make_release([], release_tag = "r3", upstream_tag = "b9003"), - checksums = make_checksums_with_source( - ["llama-b9003-bin-ubuntu-x64.tar.gz"], - release_tag = "r3", - upstream_tag = "b9003", - ), - ), - INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease( - bundle = make_release([], release_tag = "r2", upstream_tag = "b9002"), - checksums = make_checksums_with_source( - ["llama-b9002-bin-ubuntu-x64.tar.gz"], - release_tag = "r2", - upstream_tag = "b9002", - ), - ), - INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease( - bundle = make_release([], release_tag = "r1", upstream_tag = "b9001"), - checksums = make_checksums_with_source( - ["llama-b9001-bin-ubuntu-x64.tar.gz"], - release_tag = "r1", - upstream_tag = "b9001", - ), - ), + self._cuda_bundle("app-b9003-linux-x64-cuda12.tar.gz", "r3", "b9003"), + self._cuda_bundle("app-b9002-linux-x64-cuda12.tar.gz", "r2", "b9002"), + self._cuda_bundle("app-b9001-linux-x64-cuda12.tar.gz", "r1", "b9001"), ] monkeypatch.setattr( @@ -1904,15 +1904,8 @@ class TestResolveInstallReleasePlans: "iter_resolved_published_releases", lambda requested_tag, published_repo, published_release_tag = "": iter(releases), ) - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "github_release_assets", - lambda repo, tag: { - f"llama-{tag}-bin-ubuntu-x64.tar.gz": f"https://example.com/llama-{tag}-bin-ubuntu-x64.tar.gz" - }, - ) - requested_tag, plans = resolve_install_release_plans( + requested_tag, plans = _fork_manifest_release_plans( "latest", host, "unslothai/llama.cpp", @@ -1925,12 +1918,10 @@ class TestResolveInstallReleasePlans: assert [plan.llama_tag for plan in plans] == ["b9003", "b9002"] def test_latest_skips_non_installable_release_and_keeps_searching(self, monkeypatch): - host = make_host( - has_usable_nvidia = False, - has_physical_nvidia = False, - nvidia_smi = None, - ) + mock_linux_runtime(monkeypatch, ["cuda12"]) + host = make_host(system = "Linux", machine = "x86_64", compute_caps = ["86"]) releases = [ + # r2 ships no fork bundle, so it yields no plan and is skipped. INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease( bundle = make_release([], release_tag = "r2", upstream_tag = "b9002"), checksums = make_checksums_with_source( @@ -1939,14 +1930,7 @@ class TestResolveInstallReleasePlans: upstream_tag = "b9002", ), ), - INSTALL_LLAMA_PREBUILT.ResolvedPublishedRelease( - bundle = make_release([], release_tag = "r1", upstream_tag = "b9001"), - checksums = make_checksums_with_source( - ["llama-b9001-bin-ubuntu-x64.tar.gz"], - release_tag = "r1", - upstream_tag = "b9001", - ), - ), + self._cuda_bundle("app-b9001-linux-x64-cuda12.tar.gz", "r1", "b9001"), ] monkeypatch.setattr( @@ -1954,19 +1938,8 @@ class TestResolveInstallReleasePlans: "iter_resolved_published_releases", lambda requested_tag, published_repo, published_release_tag = "": iter(releases), ) - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "github_release_assets", - lambda repo, tag: ( - {} - if tag == "b9002" - else { - f"llama-{tag}-bin-ubuntu-x64.tar.gz": f"https://example.com/llama-{tag}-bin-ubuntu-x64.tar.gz" - } - ), - ) - _requested_tag, plans = resolve_install_release_plans( + _requested_tag, plans = _fork_manifest_release_plans( "latest", host, "unslothai/llama.cpp", @@ -1998,7 +1971,9 @@ class TestResolveInstallReleasePlans: sys.modules.pop(spec.name, None) +# =========================================================================== # N. windows_cuda_attempts +# =========================================================================== class TestWindowsCudaAttempts: @@ -2216,7 +2191,9 @@ class TestWindowsCudaAttempts: assert result[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip" +# =========================================================================== # N.1b. _pinned_windows_cuda_fallback -- pinned b9360 cuda-13.1 Blackwell fallback +# =========================================================================== class TestPinnedBlackwellCudaFallback: @@ -2357,8 +2334,49 @@ class TestPinnedBlackwellCudaFallback: ) assert _windows_cuda_attempt_covers_blackwell(cpu) is False + def _app_attempt(self, profile, runtime_line, max_sm): + # The fork's app-named windows-cuda bundle: no toolkit minor in the name, + # SM coverage declared directly (as published_windows_cuda_attempts sets it). + return AssetChoice( + repo = UPSTREAM_REPO, + tag = self.TAG, + name = f"app-{self.TAG}-windows-x64-{runtime_line}-{profile}.zip", + url = "https://example.com/x", + source_label = "published", + install_kind = "windows-cuda", + runtime_line = runtime_line, + coverage_class = "newer" if profile == "newer" else profile, + max_sm = max_sm, + min_sm = 80, + supported_sms = ["120"] if max_sm >= 120 else ["86", "89"], + ) + @pytest.mark.parametrize( + "profile, runtime_line, max_sm, covers", + [ + ("newer", "cuda13", 120, True), # native Blackwell build + ("newer", "cuda12", 120, True), # 12.8 toolkit app bundle reaches sm120 + ("older", "cuda12", 89, False), # 12.4 toolkit app bundle stops at Ada + ], + ) + def test_attempt_covers_blackwell_app_bundle(self, profile, runtime_line, max_sm, covers): + # App-named bundles carry no toolkit minor; coverage is read from max_sm. + attempt = self._app_attempt(profile, runtime_line, max_sm) + assert _windows_cuda_attempt_covers_blackwell(attempt) is covers + + def test_pin_dormant_when_app_bundle_covers_blackwell(self): + # Regression: the fork's app-named cuda13 bundle covers Blackwell, so the + # b9360 pin must retire instead of being prepended ahead of the native + # in-release build (previously the coverage check only matched legacy + # -bin-win-cuda-X.Y-x64.zip names, so the pin never went dormant). + host = self._win_host((13, 1), ["120"]) + existing = [self._app_attempt("newer", "cuda13", 120)] + assert _pinned_windows_cuda_fallback(host, existing) is None + + +# =========================================================================== # N.1c. direct_upstream_release_plan -- pinned Blackwell fallback ordering +# =========================================================================== class TestDirectUpstreamBlackwellPin: @@ -2593,7 +2611,9 @@ class TestDirectLinuxNvidiaCpuGate: assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] +# =========================================================================== # N.1d. published_windows_cuda_attempts -- version-dynamic ordering seed +# =========================================================================== class TestPublishedWindowsCudaAttemptsDynamicMajor: @@ -2608,6 +2628,7 @@ class TestPublishedWindowsCudaAttemptsDynamicMajor: f"llama-{self.TAG}-bin-win-cuda-{minor}-x64.zip", install_kind = "windows-cuda", runtime_line = runtime_line, + supported_sms = ["75", "80", "86", "89", "90", "100", "120"], max_sm = 120, ) @@ -2659,12 +2680,14 @@ class TestPublishedWindowsCudaAttemptsDynamicMajor: assert result[0].runtime_line == "cuda12" +# =========================================================================== # N.1e. resolve_release_asset_choice -- pin on the published install path +# =========================================================================== class TestResolveReleaseAssetChoicePin: - """The published (non --simple-policy) install path reaches the same b9360 - Blackwell pin as the simple path, with its verified hash threaded.""" + """The manifest install path reaches the same b9360 Blackwell pin as the + filename path, with its verified hash threaded.""" TAG = "b8508" @@ -2674,6 +2697,7 @@ class TestResolveReleaseAssetChoicePin: f"llama-{self.TAG}-bin-win-cuda-{minor}-x64.zip", install_kind = "windows-cuda", runtime_line = line, + supported_sms = ["75", "80", "86", "89", "90", "100", "120"], max_sm = 120, ) for minor, line in minors_lines @@ -2754,7 +2778,243 @@ class TestResolveReleaseAssetChoicePin: assert "b9360" not in [a.tag for a in result] +class TestPublishedWindowsCudaAppBundleSmSelection: + """app-named windows-cuda bundles carry no minor in the filename, so the + driver-minor gate is skipped. Selection must instead filter by SM coverage, + or every host gets the lowest-rank "older" bundle regardless of its GPU.""" + + TAG = "b9457" + + def _app(self, klass, supported, min_sm, max_sm, rank): + return make_artifact( + f"app-{self.TAG}-windows-x64-cuda12-{klass}.zip", + install_kind = "windows-cuda", + runtime_line = "cuda12", + coverage_class = klass, + supported_sms = supported, + min_sm = min_sm, + max_sm = max_sm, + bundle_profile = f"cuda12-{klass}", + rank = rank, + ) + + def test_blackwell_sm120_skips_older_bundle(self, monkeypatch): + mock_windows_runtime(monkeypatch, ["cuda12"]) + older = self._app("older", ["70", "75", "80", "86", "89"], 70, 89, 10) + newer = self._app("newer", ["86", "89", "90", "100", "120"], 86, 120, 20) + portable = self._app( + "portable", ["70", "75", "80", "86", "89", "90", "100", "120"], 70, 120, 30 + ) + release = make_release([older, newer, portable], upstream_tag = self.TAG) + host = make_host( + system = "Windows", + machine = "AMD64", + driver_cuda_version = (12, 8), + compute_caps = ["120"], + ) + result = published_windows_cuda_attempts(host, release, None) + assert result, "expected a windows-cuda attempt for an sm120 host" + # The lowest-rank "older" bundle (max_sm 89) must not be chosen, and the + # tightest covering bundle is cuda12-newer (range 86-120). + assert result[0].name == f"app-{self.TAG}-windows-x64-cuda12-newer.zip" + + def _line(self, line, klass, rank): + return make_artifact( + f"app-{self.TAG}-windows-x64-{line}-{klass}.zip", + install_kind = "windows-cuda", + runtime_line = line, + coverage_class = klass, + supported_sms = ["86", "89", "90", "100", "120"], + min_sm = 86, + max_sm = 120, + bundle_profile = f"{line}-{klass}", + rank = rank, + ) + + def test_cuda13_reachable_on_driver_13_0(self, monkeypatch): + # app-named cuda13 bundles must be reachable on a 13.0 driver. The old + # synthetic '13.1' minor gate dropped the whole cuda13 line (13.1 > 13.0), + # so a cu13 host fell to cuda12. cuda13 is gated at the major level now. + mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"]) + release = make_release( + [self._line("cuda12", "newer", 20), self._line("cuda13", "newer", 50)], + upstream_tag = self.TAG, + ) + host = make_host( + system = "Windows", + machine = "AMD64", + driver_cuda_version = (13, 0), + compute_caps = ["120"], + ) + result = published_windows_cuda_attempts(host, release, "cuda13") + assert result + assert result[0].runtime_line == "cuda13" + assert result[0].name == f"app-{self.TAG}-windows-x64-cuda13-newer.zip" + + def test_app_bundle_offered_when_no_runtime_dll_detected(self, monkeypatch): + # Windows torch bundles cudart in torch/lib, which runtime-DLL probing + # misses, so detected_windows_runtime_lines() returns nothing. The app + # bundle ships its own runtime, so selection must fall back to the + # driver-derived order instead of yielding no attempt (which would drop + # the host to the upstream build). + mock_windows_runtime(monkeypatch, []) + release = make_release( + [self._line("cuda12", "newer", 20), self._line("cuda13", "newer", 50)], + upstream_tag = self.TAG, + ) + host = make_host( + system = "Windows", + machine = "AMD64", + driver_cuda_version = (13, 0), + compute_caps = ["120"], + ) + result = published_windows_cuda_attempts(host, release, "cuda13") + assert result, "torch-only host must still get the fork app bundle" + assert result[0].name == f"app-{self.TAG}-windows-x64-cuda13-newer.zip" + + +class TestPublishedRocmGfxSelection: + """Published ROCm bundles are matched by the host's detected gfx family, not + by rank -- rank ties would alphabetically hand every AMD GPU the gfx103X + bundle (e.g. a gfx1151 Strix Halo host).""" + + GFX = ["gfx103X", "gfx110X", "gfx120X", "gfx1150", "gfx1151"] + MEMBERS = { + "gfx103X": ["gfx1030", "gfx1031", "gfx1032", "gfx1034"], + "gfx110X": ["gfx1100", "gfx1101", "gfx1102", "gfx1103"], + "gfx120X": ["gfx1200", "gfx1201"], + "gfx1150": ["gfx1150"], + "gfx1151": ["gfx1151"], + } + + def _release(self, install_kind, prefix): + artifacts = [ + make_artifact( + f"{prefix}-{gfx}.{'zip' if 'windows' in install_kind else 'tar.gz'}", + install_kind = install_kind, + runtime_line = None, + coverage_class = None, + supported_sms = [], + min_sm = None, + max_sm = None, + bundle_profile = None, + rank = 1000, + gfx_target = gfx, + mapped_targets = self.MEMBERS[gfx], + ) + for gfx in self.GFX + ] + return make_release(artifacts, upstream_tag = "b9457") + + def _host(self, gfx): + return make_host( + machine = "x86_64", + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + has_physical_nvidia = False, + has_usable_nvidia = False, + has_rocm = True, + rocm_gfx_target = gfx, + ) + + def test_gfx1100_selects_gfx110X_family(self): + release = self._release("linux-rocm", "app-b9457-linux-x64-rocm") + choice = INSTALL_LLAMA_PREBUILT.published_rocm_choice_for_host( + release, self._host("gfx1100"), "linux-rocm" + ) + assert choice is not None + assert choice.name == "app-b9457-linux-x64-rocm-gfx110X.tar.gz" + + def test_gfx1151_strix_halo_not_handed_gfx103X(self): + release = self._release("linux-rocm", "app-b9457-linux-x64-rocm") + choice = INSTALL_LLAMA_PREBUILT.published_rocm_choice_for_host( + release, self._host("gfx1151"), "linux-rocm" + ) + assert choice is not None + assert choice.name == "app-b9457-linux-x64-rocm-gfx1151.tar.gz" + + def test_windows_rocm_gfx_match(self): + release = self._release("windows-rocm", "app-b9457-windows-x64-rocm") + choice = INSTALL_LLAMA_PREBUILT.published_rocm_choice_for_host( + release, self._host("gfx1201"), "windows-rocm" + ) + assert choice is not None + assert choice.name == "app-b9457-windows-x64-rocm-gfx120X.zip" + + def test_uncovered_gpu_returns_none(self): + release = self._release("linux-rocm", "app-b9457-linux-x64-rocm") + assert ( + INSTALL_LLAMA_PREBUILT.published_rocm_choice_for_host( + release, self._host("gfx900"), "linux-rocm" + ) + is None + ) + + def test_in_prefix_but_unbuilt_arch_returns_none(self): + # gfx1033 shares the gfx103 prefix but is not in any bundle's + # mapped_targets, so it must fall back to source, not be served gfx103X. + release = self._release("linux-rocm", "app-b9457-linux-x64-rocm") + for unbuilt in ("gfx1033", "gfx1035", "gfx1104", "gfx1202"): + assert ( + INSTALL_LLAMA_PREBUILT.published_rocm_choice_for_host( + release, self._host(unbuilt), "linux-rocm" + ) + is None + ), unbuilt + + +class TestPublishedMacosForkSelection: + """macOS now routes to the fork (setup.sh), which ships + llama--bin-macos-.tar.gz with pinned deployment targets, selected + by install_kind.""" + + def _release(self): + arts = [ + make_artifact( + "llama-b9457-bin-macos-arm64.tar.gz", + install_kind = "macos-arm64", + runtime_line = None, + coverage_class = None, + supported_sms = [], + min_sm = None, + max_sm = None, + bundle_profile = "macos-metal-arm64", + rank = 50, + ), + make_artifact( + "llama-b9457-bin-macos-x64.tar.gz", + install_kind = "macos-x64", + runtime_line = None, + coverage_class = None, + supported_sms = [], + min_sm = None, + max_sm = None, + bundle_profile = "macos-cpu-x64", + rank = 50, + ), + ] + return make_release(arts, upstream_tag = "b9457") + + def test_macos_arm64_selects_fork_bundle(self): + choice = INSTALL_LLAMA_PREBUILT.published_asset_choice_for_kind( + self._release(), "macos-arm64" + ) + assert choice is not None + assert choice.name == "llama-b9457-bin-macos-arm64.tar.gz" + assert choice.install_kind == "macos-arm64" + + def test_macos_x64_selects_fork_bundle(self): + choice = INSTALL_LLAMA_PREBUILT.published_asset_choice_for_kind( + self._release(), "macos-x64" + ) + assert choice is not None + assert choice.name == "llama-b9457-bin-macos-x64.tar.gz" + + +# =========================================================================== # N.1. apply_approved_hashes -- runtime archive checksum threading +# =========================================================================== class TestApplyApprovedHashesRuntimePair: @@ -2826,7 +3086,9 @@ class TestApplyApprovedHashesRuntimePair: assert result[0].runtime_sha256 is None +# =========================================================================== # O. resolve_upstream_asset_choice -- platform routing +# =========================================================================== class TestResolveUpstreamAssetChoice: @@ -2969,7 +3231,9 @@ class TestResolveUpstreamAssetChoice: assert result.name == cuda_name +# =========================================================================== # N.2. Deterministic macOS prebuilt pin (b9415) +# =========================================================================== def _macos_host(machine = "arm64", version = (15, 5)): @@ -3092,37 +3356,47 @@ class TestResolveSimpleMacosPin: assert calls[0][2] == "latest" +# =========================================================================== # Linux arm64 + GPU must not install the x64-only fork bundle +# =========================================================================== class TestLinuxArm64ForkFallsBackToSource: - """The unslothai/llama.cpp fork ships only linux-x64 bundles. An arm64 - Linux host with a GPU (GH200/GB200/DGX Spark) routes to the fork and must - fall back to a source build instead of selecting an x64 binary.""" + """The fork now ships linux-arm64-cuda bundles (GH200/GB200/DGX Spark). An + arm64 Linux host on the fork no longer hard-fails on the simple path; it + delegates to the manifest-aware resolver, which selects the arm64 CUDA + bundle (or falls back to source only if none matches).""" - def test_arm64_nvidia_fork_raises_before_fetching_releases(self, monkeypatch): - # Guard fires before any release is fetched: poison the iterator to prove - # it is never called. - def _boom(*_a, **_k): - raise AssertionError("iterator must not run for arm64 fork hosts") + def test_arm64_nvidia_fork_delegates_to_manifest_resolver(self, monkeypatch): + # arm64 fork hosts are no longer blocked up front; the simple resolver + # hands them to the manifest-aware resolver instead. + called = {} - monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "iter_release_payloads_by_time", _boom) + def _full(llama_tag, host, repo, tag, **_kw): + called["args"] = (host.machine, repo) + return "b9457", ["plan"] + + monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_fork_manifest_release_plans", _full) host = make_host(system = "Linux", machine = "aarch64") - with pytest.raises(PrebuiltFallback, match = "linux-x64 prebuilts"): - resolve_simple_install_release_plans("latest", host, "unslothai/llama.cpp", "") + tag, plans = resolve_simple_install_release_plans("latest", host, "unslothai/llama.cpp", "") + assert called.get("args") == ("aarch64", "unslothai/llama.cpp") + assert plans == ["plan"] - def test_x86_64_fork_is_not_blocked_by_the_arch_guard(self, monkeypatch): - # x64 host must pass the guard and reach the iterator (here empty, so it - # raises the generic message, not the arch one). - monkeypatch.setattr( - INSTALL_LLAMA_PREBUILT, - "iter_release_payloads_by_time", - lambda *_a, **_k: iter(()), - ) + def test_x86_64_fork_delegates_to_manifest_resolver(self, monkeypatch): + # The old linux-x64 arch guard is gone: an x64 fork host is routed to the + # manifest resolver exactly like every other fork host, not down a + # separate filename-parsing path. + called = {} + + def _full(llama_tag, host, repo, tag, **_kw): + called["args"] = (host.machine, repo) + return "b9457", ["plan"] + + monkeypatch.setattr(INSTALL_LLAMA_PREBUILT, "_fork_manifest_release_plans", _full) host = make_host(system = "Linux", machine = "x86_64") - with pytest.raises(PrebuiltFallback) as exc: - resolve_simple_install_release_plans("latest", host, "unslothai/llama.cpp", "") - assert "linux-x64 prebuilts" not in str(exc.value) + tag, plans = resolve_simple_install_release_plans("latest", host, "unslothai/llama.cpp", "") + assert called.get("args") == ("x86_64", "unslothai/llama.cpp") + assert plans == ["plan"] def test_arm64_cpu_on_ggml_org_is_not_blocked(self, monkeypatch): # CPU-only arm64 routes to ggml-org (not the fork), so the guard must not @@ -3146,7 +3420,9 @@ class TestLinuxArm64ForkFallsBackToSource: assert "linux-x64 prebuilts" not in str(exc.value) +# =========================================================================== # arm64 Linux GPU: CPU prebuilt fallback after a failed source build (--cpu-fallback) +# =========================================================================== class TestCpuFallback: @@ -3187,7 +3463,6 @@ class TestCpuFallback: llama_tag = "latest", published_repo = "ggml-org/llama.cpp", published_release_tag = "", - simple_policy = True, force_cpu = True, ) host = captured["host"]