From cc904e9b640139d01ffd0724e205b82d57bb30cf Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 08:30:53 +0000 Subject: [PATCH] Add ROCm support to llama.cpp prebuilt installer Extend the prebuilt installer to detect AMD ROCm GPUs and select the appropriate llama.cpp binary: - Add has_rocm field (default False) to HostInfo dataclass - Extend detect_host() to probe for ROCm via hipcc, amd-smi, rocm-smi, /opt/rocm, and ROCM_PATH env var (skipped on macOS) - Linux x86_64 ROCm hosts (without NVIDIA) try the upstream ROCm 7.2 prebuilt first, with a log warning that it may fall back to source build on other ROCm versions - Windows x86_64 ROCm hosts try the HIP prebuilt; if not found, a log message is printed before falling through to CPU - Add "linux-rocm" and "windows-hip" install kinds to runtime_patterns_for_choice() with the correct shared library globs (libggml-hip.so* for Linux, *.dll for Windows) The ROCm path is only entered when has_usable_nvidia is False, so NVIDIA always takes precedence on dual-GPU systems. The source build fallback (via setup.sh with -DGGML_HIP=ON) compiles against the exact GPU target via rocminfo, which is more reliable for consumer GPUs (e.g. gfx1151) that may not be in the prebuilt. --- studio/install_llama_prebuilt.py | 58 ++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index 516dc4b6a4..cbc8f6d15f 100755 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -88,6 +88,7 @@ class HostInfo: visible_cuda_devices: str | None has_physical_nvidia: bool has_usable_nvidia: bool + has_rocm: bool = False @dataclass @@ -1430,6 +1431,18 @@ def detect_host() -> HostInfo: except Exception: pass + # Detect AMD ROCm (HIP) + has_rocm = False + if not is_macos: + rocm_hints = [ + shutil.which("hipcc"), + shutil.which("amd-smi"), + shutil.which("rocm-smi"), + ] + rocm_paths = ["/opt/rocm", os.environ.get("ROCM_PATH", "")] + if any(rocm_hints) or any(os.path.isdir(p) for p in rocm_paths if p): + has_rocm = True + return HostInfo( system = system, machine = machine, @@ -1444,6 +1457,7 @@ def detect_host() -> HostInfo: visible_cuda_devices = visible_cuda_devices, has_physical_nvidia = has_physical_nvidia, has_usable_nvidia = has_usable_nvidia, + has_rocm = has_rocm, ) @@ -1724,6 +1738,30 @@ def resolve_linux_cuda_choice( def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice: upstream_assets = github_release_assets(UPSTREAM_REPO, llama_tag) if host.is_linux and host.is_x86_64: + # AMD ROCm: try upstream ROCm prebuilt first, then fall back to source build. + # Source build (via setup.sh) compiles with -DGGML_HIP=ON and auto-detects + # the exact GPU target via rocminfo, which is more reliable for consumer + # GPUs (e.g. gfx1151) that may not be in the prebuilt. + if host.has_rocm and not host.has_usable_nvidia: + rocm_name = f"llama-{llama_tag}-bin-ubuntu-rocm-7.2-x64.tar.gz" + if rocm_name in upstream_assets: + log(f"AMD ROCm detected -- trying upstream prebuilt {rocm_name}") + log("Note: prebuilt is compiled for ROCm 7.2; if your ROCm version differs, " + "this may fail preflight and fall back to a source build (safe)") + return AssetChoice( + repo = UPSTREAM_REPO, + tag = llama_tag, + name = rocm_name, + url = upstream_assets[rocm_name], + source_label = "upstream", + install_kind = "linux-rocm", + ) + # No ROCm prebuilt available -- fall back to source build + raise PrebuiltFallback( + "AMD ROCm detected but no upstream ROCm prebuilt found; " + "falling back to source build with HIP support" + ) + upstream_name = f"llama-{llama_tag}-bin-ubuntu-x64.tar.gz" if upstream_name not in upstream_assets: raise PrebuiltFallback("upstream Linux CPU asset was not found") @@ -1743,6 +1781,21 @@ def resolve_upstream_asset_choice(host: HostInfo, llama_tag: str) -> AssetChoice return attempts[0] raise PrebuiltFallback("no compatible Windows CUDA asset was found") + # AMD ROCm on Windows: try HIP prebuilt + if host.has_rocm: + hip_name = f"llama-{llama_tag}-bin-win-hip-radeon-x64.zip" + if hip_name in upstream_assets: + log(f"AMD ROCm detected on Windows -- trying upstream HIP prebuilt {hip_name}") + return AssetChoice( + repo = UPSTREAM_REPO, + tag = llama_tag, + name = hip_name, + url = upstream_assets[hip_name], + source_label = "upstream", + install_kind = "windows-hip", + ) + log("AMD ROCm detected on Windows but no HIP prebuilt found -- falling back to CPU") + upstream_name = f"llama-{llama_tag}-bin-win-cpu-x64.zip" if upstream_name not in upstream_assets: raise PrebuiltFallback("upstream Windows CPU asset was not found") @@ -2121,7 +2174,7 @@ def overlay_directory_for_choice( def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: - if choice.install_kind in {"linux-cpu", "linux-cuda"}: + if choice.install_kind in {"linux-cpu", "linux-cuda", "linux-rocm"}: return [ "llama-server", "llama-quantize", @@ -2131,11 +2184,12 @@ def runtime_patterns_for_choice(choice: AssetChoice) -> list[str]: "libmtmd.so*", "libggml-cpu-*.so*", "libggml-cuda.so*", + "libggml-hip.so*", "libggml-rpc.so*", ] if choice.install_kind in {"macos-arm64", "macos-x64"}: return ["llama-server", "llama-quantize", "lib*.dylib"] - if choice.install_kind in {"windows-cpu", "windows-cuda"}: + if choice.install_kind in {"windows-cpu", "windows-cuda", "windows-hip"}: return ["*.exe", "*.dll"] raise PrebuiltFallback( f"unsupported install kind for runtime overlay: {choice.install_kind}"