diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 4509ea58ed..bc4fe555c1 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -1410,10 +1410,16 @@ def direct_linux_release_plan( ) if lemonade_choice is not None: attempts.append(lemonade_choice) - else: + elif not host.has_usable_nvidia: cpu_choice = published_asset_choice_for_kind(bundle, "linux-cpu") if cpu_choice is not None: attempts.append(cpu_choice) + # NVIDIA hosts whose CUDA selection produced nothing fall through to the + # raise below (mirroring the ROCm policy above): the caller then walks + # back to an older release that still ships a usable CUDA line instead of + # silently installing a CPU binary on a GPU host. Today's walk-back only + # works because partial releases ship no CPU bundle; this keeps it working + # if a future partial release does. if not attempts: raise PrebuiltFallback("no compatible Linux prebuilt asset was found") approved_checksums = synthetic_checksums_for_release( @@ -1476,6 +1482,7 @@ def direct_upstream_release_plan( torch_preference.selection_log, ) ) + attempts[:] = _drop_blackwell_incapable_windows_cuda(host, attempts) # Blackwell on a 13.1/13.2 driver: prefer the pinned cuda-13.1 GPU # build over the CPU-only cuda-12.4 left by in-release gating. pinned = _pinned_windows_cuda_fallback(host, attempts) @@ -3315,14 +3322,49 @@ def windows_cuda_attempts( def _windows_cuda_attempt_covers_blackwell(attempt: AssetChoice) -> bool: - """True if an in-release windows-cuda attempt's toolkit covers Blackwell - sm_120 (>= 12.8), read from its asset name's CUDA minor.""" + """True if an in-release windows-cuda attempt covers Blackwell sm_120. + + Manifest-backed app bundles carry their compiled SM list, so trust that + first (the published cuda12 bundles are toolkit-12.8 builds that include + sm_120 even though their runtime line says cuda12). Upstream ggml-org + zips have no manifest metadata; infer from the CUDA minor in the asset + name (sm_120 needs toolkit >= 12.8).""" if attempt.install_kind != "windows-cuda": return False + if attempt.max_sm is not None: + return attempt.max_sm >= _BLACKWELL_MIN_SM + if attempt.supported_sms: + sms = normalize_compute_caps(attempt.supported_sms) + if sms: + return int(sms[-1]) >= _BLACKWELL_MIN_SM m = re.search(r"-bin-win-cuda-(\d+)\.(\d+)-x64\.zip$", attempt.name) return m is not None and (int(m.group(1)), int(m.group(2))) >= _BLACKWELL_MIN_TOOLKIT +def _host_is_blackwell(host: HostInfo) -> bool: + caps = normalize_compute_caps(host.compute_caps) + return bool(caps) and int(caps[-1]) >= _BLACKWELL_MIN_SM + + +def _drop_blackwell_incapable_windows_cuda( + host: HostInfo, attempts: list[AssetChoice] +) -> list[AssetChoice]: + """On a Blackwell host, drop windows-cuda attempts that cannot offload + sm_120 (e.g. upstream cuda-12.4, toolkit 12.4). Such a build loads and + passes the functional validator but runs the model on a slow non-native + path (an RTX 5090 measured 7.1 tok/s vs 551.2 on cuda-13.3), so it must + not sit in the fallback chain behind the pin or an in-release cuda13. + Non-cuda attempts (windows-cpu, windows-hip, ...) pass through so the + host still degrades to an honest CPU install when no CUDA 13 exists.""" + if not _host_is_blackwell(host): + return attempts + return [ + attempt + for attempt in attempts + if attempt.install_kind != "windows-cuda" or _windows_cuda_attempt_covers_blackwell(attempt) + ] + + def _pinned_windows_cuda_fallback( host: HostInfo, existing_cuda_attempts: list[AssetChoice] ) -> AssetChoice | None: @@ -3405,6 +3447,7 @@ def _with_pinned_windows_cuda_fallback( """Insert the Blackwell pin ahead of the Windows CUDA attempts and keep it through apply_approved_hashes, or return inputs unchanged when dormant. Gives the published install path the same GPU fallback as the simple path.""" + attempts = _drop_blackwell_incapable_windows_cuda(host, attempts) pin = _pinned_windows_cuda_fallback(host, attempts) if pin is None: return attempts, checksums diff --git a/tests/studio/install/test_selection_logic.py b/tests/studio/install/test_selection_logic.py index b5a070ae41..dfa2378ef5 100644 --- a/tests/studio/install/test_selection_logic.py +++ b/tests/studio/install/test_selection_logic.py @@ -2401,7 +2401,10 @@ class TestDirectUpstreamBlackwellPin: ) plan = direct_upstream_release_plan(self._release(), host, UPSTREAM_REPO, "latest") order = [(a.tag, a.runtime_line or a.install_kind) for a in plan.attempts] - assert order == [("b9360", "cuda13"), (self.TAG, "cuda12"), (self.TAG, "windows-cpu")] + # cuda-12.4 (toolkit 12.4, no sm_120) is dropped entirely on Blackwell: + # behind the pin it would still be attempted if the pin download failed, + # and the functional validator accepts its slow non-native path. + assert order == [("b9360", "cuda13"), (self.TAG, "windows-cpu")] assert plan.attempts[0].name == "llama-b9360-bin-win-cuda-13.1-x64.zip" # Direct/upstream path stays unverified-by-manifest (no approved hashes). assert plan.approved_checksums.artifacts == {} @@ -2422,6 +2425,174 @@ class TestDirectUpstreamBlackwellPin: assert plan.attempts[0].name == f"llama-{self.TAG}-bin-win-cuda-13.3-x64.zip" +# N.1c2. Blackwell never falls to a non-sm_120 windows-cuda attempt + + +class TestBlackwellCuda124Exclusion: + """A Blackwell host must never have a windows-cuda attempt that cannot + offload sm_120 anywhere in its chain: behind the pin it is one failed + download away from a validated-but-7-tok/s install.""" + + def _bw_host(self): + return make_host( + system = "Windows", + machine = "AMD64", + driver_cuda_version = (13, 1), + compute_caps = ["120"], + ) + + def _upstream_cuda( + self, + minor, + tag = "b9365", + ): + return AssetChoice( + repo = "ggml-org/llama.cpp", + tag = tag, + name = f"llama-{tag}-bin-win-cuda-{minor}-x64.zip", + url = f"https://example.com/{minor}", + source_label = "upstream", + install_kind = "windows-cuda", + runtime_line = "cuda" + minor.split(".")[0], + ) + + def test_drops_124_keeps_133_on_blackwell(self): + kept = INSTALL_LLAMA_PREBUILT._drop_blackwell_incapable_windows_cuda( + self._bw_host(), + [self._upstream_cuda("13.3"), self._upstream_cuda("12.4")], + ) + assert [a.name for a in kept] == ["llama-b9365-bin-win-cuda-13.3-x64.zip"] + + def test_keeps_manifest_cuda12_bundle_with_sm120(self): + # Published cuda12 app bundles are toolkit-12.8 builds that include + # sm_120; the manifest SM metadata must keep them on Blackwell. + bundle = AssetChoice( + repo = "unslothai/llama.cpp", + tag = "b9585", + name = "app-b9585-windows-x64-cuda12-portable.zip", + url = "https://example.com/app", + source_label = "published", + install_kind = "windows-cuda", + runtime_line = "cuda12", + supported_sms = ["70", "120"], + max_sm = 120, + ) + kept = INSTALL_LLAMA_PREBUILT._drop_blackwell_incapable_windows_cuda( + self._bw_host(), [bundle] + ) + assert kept == [bundle] + assert _windows_cuda_attempt_covers_blackwell(bundle) + + def test_manifest_bundle_without_sm120_dropped(self): + bundle = AssetChoice( + repo = "unslothai/llama.cpp", + tag = "b9585", + name = "app-b9585-windows-x64-cuda12-older.zip", + url = "https://example.com/app", + source_label = "published", + install_kind = "windows-cuda", + runtime_line = "cuda12", + supported_sms = ["70", "75", "80"], + max_sm = 80, + ) + assert ( + INSTALL_LLAMA_PREBUILT._drop_blackwell_incapable_windows_cuda(self._bw_host(), [bundle]) + == [] + ) + + def test_non_blackwell_host_unfiltered(self): + host = make_host( + system = "Windows", + machine = "AMD64", + driver_cuda_version = (12, 9), + compute_caps = ["89"], + ) + attempts = [self._upstream_cuda("12.4")] + assert ( + INSTALL_LLAMA_PREBUILT._drop_blackwell_incapable_windows_cuda(host, attempts) + == attempts + ) + + def test_non_cuda_attempts_pass_through(self): + cpu = AssetChoice( + repo = "ggml-org/llama.cpp", + tag = "b9365", + name = "llama-b9365-bin-win-cpu-x64.zip", + url = "https://example.com/cpu", + source_label = "upstream", + install_kind = "windows-cpu", + ) + kept = INSTALL_LLAMA_PREBUILT._drop_blackwell_incapable_windows_cuda( + self._bw_host(), [self._upstream_cuda("12.4"), cpu] + ) + assert kept == [cpu] + + +# N.1c3. direct_linux_release_plan -- no silent CPU on NVIDIA hosts + + +class TestDirectLinuxNvidiaCpuGate: + """When a release ships a linux-cpu bundle but no CUDA line this NVIDIA + host can use, the planner must raise (so the caller walks back to an older + release with a usable CUDA line) instead of silently planning a CPU + install on a GPU host. CPU-only hosts keep taking the CPU bundle.""" + + def _bundle_cpu_only(self): + return make_release( + [ + make_artifact( + "llama-b8508-bin-ubuntu-x64.tar.gz", + install_kind = "linux-cpu", + runtime_line = None, + coverage_class = None, + supported_sms = [], + min_sm = None, + max_sm = None, + bundle_profile = None, + ), + ] + ) + + def _patch(self, monkeypatch): + monkeypatch.setattr( + INSTALL_LLAMA_PREBUILT, + "parse_direct_linux_release_bundle", + lambda repo, release: self._bundle_cpu_only(), + ) + monkeypatch.setattr( + INSTALL_LLAMA_PREBUILT, + "detect_torch_cuda_runtime_preference", + lambda host: CudaRuntimePreference(runtime_line = None, selection_log = []), + ) + monkeypatch.setattr( + INSTALL_LLAMA_PREBUILT, + "detected_linux_runtime_lines", + lambda: (["cuda13"], {"cuda13": ["/usr/local/cuda/lib64"]}), + ) + + def test_nvidia_host_without_cuda_line_raises_for_walkback(self, monkeypatch): + self._patch(monkeypatch) + host = make_host(driver_cuda_version = (13, 1), compute_caps = ["100"]) + with pytest.raises(PrebuiltFallback, match = "no compatible Linux prebuilt"): + INSTALL_LLAMA_PREBUILT.direct_linux_release_plan( + {"tag_name": "b8508"}, host, "unslothai/llama.cpp", "latest" + ) + + def test_cpu_host_still_gets_cpu_bundle(self, monkeypatch): + self._patch(monkeypatch) + host = make_host( + nvidia_smi = None, + driver_cuda_version = None, + compute_caps = [], + has_physical_nvidia = False, + has_usable_nvidia = False, + ) + plan = INSTALL_LLAMA_PREBUILT.direct_linux_release_plan( + {"tag_name": "b8508"}, host, "unslothai/llama.cpp", "latest" + ) + assert [a.install_kind for a in plan.attempts] == ["linux-cpu"] + + # N.1d. published_windows_cuda_attempts -- version-dynamic ordering seed @@ -2549,7 +2720,9 @@ class TestResolveReleaseAssetChoicePin: # augmented checksums (the pin survives the approved-hash gate). assert result[0].expected_sha256 and len(result[0].expected_sha256) == 64 assert result[0].runtime_sha256 and len(result[0].runtime_sha256) == 64 - assert any(a.runtime_line == "cuda12" for a in result) + # The sm_120-incapable upstream cuda-12.4 zip is excluded on Blackwell + # rather than left behind the pin as a slow-path fallback. + assert not any(a.runtime_line == "cuda12" for a in result) def test_pin_dormant_on_published_path_for_13_3(self, monkeypatch): mock_windows_runtime(monkeypatch, ["cuda13", "cuda12"])