From c1947ed9465e2b6959fac19e07da95b9e7dda361 Mon Sep 17 00:00:00 2001 From: Ayushman <139611211+InfoSage05@users.noreply.github.com> Date: Wed, 22 Jul 2026 06:30:22 +0530 Subject: [PATCH] fix(rocm): prepend system ROCm libs on native Linux to avoid bundled HIP crash (#7233) * fix(rocm): prepend system ROCm libs on native Linux to avoid bundled HIP crash Prebuilt llama.cpp bundles ship their own ROCR/HIP runtime which can be incompatible with the host's amdkfd kernel driver, causing hsa_init() to crash or report zero devices. The llama-server then silently falls back to CPU while the UI reports GPU. The existing workaround (_wsl_system_rocm_lib_dirs) that prepends /opt/rocm/lib to LD_LIBRARY_PATH was gated on WSL (/dev/dxg) only, leaving native Linux AMD hosts unprotected. This commit adds _native_linux_system_rocm_lib_dirs(), a parallel helper gated on: - Linux platform (not WSL) - /dev/kfd present (bare-metal AMD compute) - Bundle contains bundled HIP libs (libggml-hip.so) - System has libhsa-runtime64.so(.1) It is called from both _llama_server_env_for_binary (serve-time) and binary_env (install-time validation), directly after the WSL block in both paths. Fixes #7208 Fixes #7208 * Add UNSLOTH_LLAMA_NO_SYSTEM_ROCM opt-out to native-Linux system ROCm preference for PR #7233 Lets a host where the bundled runtime works but system ROCm is mismatched keep the bundle. Mirrored in llama_cpp.py and install_llama_prebuilt.py. * Prefer env-configured ROCm root over /opt/rocm fallback for PR #7233 Put HIP_PATH/HIP_PATH_57/ROCM_PATH-derived roots before /opt/rocm so a stale /opt/rocm can't shadow the driver-matching install the env vars point at. Mirrored in llama_cpp.py and install_llama_prebuilt.py. * Match versioned libggml-hip.so via glob so the native-Linux ROCm fix fires for PR #7233 * Clarify native-Linux ROCm prepend uses the consistent system stack for PR #7233 * llama_cpp: tighten native-Linux ROCm prepend comments (no code change) --------- Co-authored-by: Daniel Han --- studio/backend/core/inference/llama_cpp.py | 56 ++++++++++++++++++++++ studio/install_llama_prebuilt.py | 49 +++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 2c7433f7a4..8651ed9ea8 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -247,6 +247,59 @@ def _wsl_system_rocm_lib_dirs() -> "list[str]": return out +def _bundled_hip_present(binary_dir: str) -> bool: + """True when a prebuilt bundle ships its own HIP backend library.""" + if not binary_dir: + return False + try: + # Glob the version suffix (libggml-hip.so, .so.0, .so.0.11.1) the same + # way the installer's runtime health check matches libggml-hip.so*. + return any(Path(str(binary_dir)).glob("libggml-hip.so*")) + except OSError: + return False + + +def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> "list[str]": + """System ROCm lib dir(s) to prepend before a prebuilt's bundled HIP, on native Linux. + + The bundled bare-metal HIP runtime can mismatch the host amdkfd driver and crash + in hsa_init(); prepending the whole system ROCm lib dir loads a driver-matched, + version-consistent stack (libhsa-runtime64 / libamdhip64 / librocblas) ahead of it. + The whole dir is deliberate: mixing the bundle's rocBLAS with a different-version + system HIP/ROCR risks missing symbols. UNSLOTH_LLAMA_NO_SYSTEM_ROCM=1 keeps the pure + bundle (for a host whose system ROCm lacks this arch); no-op on WSL / non-Linux. + """ + if os.environ.get("UNSLOTH_LLAMA_NO_SYSTEM_ROCM") == "1": + return [] + if sys.platform != "linux" or os.path.exists("/dev/dxg"): + return [] + if not os.path.exists("/dev/kfd"): + return [] + if not _bundled_hip_present(binary_dir): + return [] + # Env-configured ROCm root first; /opt/rocm only as a fallback so a stale + # /opt/rocm doesn't shadow the driver-matching install these vars point at. + candidates = [] + for var in ("HIP_PATH", "HIP_PATH_57", "ROCM_PATH"): + val = os.environ.get(var) + if val: + candidates.append(val) + candidates.append("/opt/rocm") + out: "list[str]" = [] + seen: "set[str]" = set() + for base in candidates: + for lib_sub in ("lib", "lib64"): + d = os.path.join(base, lib_sub) + if d in seen: + continue + seen.add(d) + if os.path.exists(os.path.join(d, "libhsa-runtime64.so")) or os.path.exists( + os.path.join(d, "libhsa-runtime64.so.1") + ): + out.append(d) + return out + + # Plan-without-action re-prompt state now lives in tool_call_parser (imported above). # Default max_tokens to the effective context when known. The floor is high @@ -3592,6 +3645,9 @@ class LlamaCppBackend: lib_dirs.extend(_wsl_system_rocm_lib_dirs()) if lib_dirs: env.setdefault("HSA_ENABLE_DXG_DETECTION", "1") + # Native Linux AMD: system ROCm libs before the bundle's HIP runtime, + # which can be incompatible with the host amdkfd driver. + lib_dirs.extend(_native_linux_system_rocm_lib_dirs(binary_dir)) lib_dirs.append(binary_dir) _arch = platform.machine() # x86_64, aarch64, etc. diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py index b8182a534b..1093ae2fd0 100644 --- a/studio/install_llama_prebuilt.py +++ b/studio/install_llama_prebuilt.py @@ -5717,6 +5717,50 @@ def _wsl_system_rocm_lib_dirs() -> list[str]: return out +def _bundled_hip_present(binary_dir: str) -> bool: + if not binary_dir: + return False + try: + # Glob the version suffix (libggml-hip.so, .so.0, .so.0.11.1) the same + # way the installer's runtime health check matches libggml-hip.so*. + return any(Path(str(binary_dir)).glob("libggml-hip.so*")) + except OSError: + return False + + +def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> list[str]: + # UNSLOTH_LLAMA_NO_SYSTEM_ROCM=1 keeps the bundled runtime (opt-out). + if os.environ.get("UNSLOTH_LLAMA_NO_SYSTEM_ROCM") == "1": + return [] + if sys.platform != "linux" or os.path.exists("/dev/dxg"): + return [] + if not os.path.exists("/dev/kfd"): + return [] + if not _bundled_hip_present(binary_dir): + return [] + # Env-configured ROCm root first; /opt/rocm only as a fallback so a stale + # /opt/rocm doesn't shadow the driver-matching install these vars point at. + candidates = [] + for var in ("HIP_PATH", "HIP_PATH_57", "ROCM_PATH"): + val = os.environ.get(var) + if val: + candidates.append(val) + candidates.append("/opt/rocm") + out: list[str] = [] + seen: set[str] = set() + for base in candidates: + for lib_sub in ("lib", "lib64"): + d = os.path.join(base, lib_sub) + if d in seen: + continue + seen.add(d) + if os.path.exists(os.path.join(d, "libhsa-runtime64.so")) or os.path.exists( + os.path.join(d, "libhsa-runtime64.so.1") + ): + out.append(d) + return out + + # Secrets a downloaded llama.cpp binary never needs; keep them out of binary_env(). # The installer's own API calls read os.environ directly, so auth is unaffected. _SECRET_ENV_EXACT_NAMES = frozenset( @@ -5877,6 +5921,11 @@ def binary_env( if _wsl_rocm: ld_dirs = [*_wsl_rocm, *ld_dirs] env.setdefault("HSA_ENABLE_DXG_DETECTION", "1") + # Native Linux AMD: system ROCm libs before the bundle's bundled HIP + # runtime, which can be incompatible with the host amdkfd driver. + _native_rocm = _native_linux_system_rocm_lib_dirs(str(binary_path.parent)) + if _native_rocm: + ld_dirs = [*_native_rocm, *ld_dirs] existing = [part for part in env.get("LD_LIBRARY_PATH", "").split(os.pathsep) if part] env["LD_LIBRARY_PATH"] = os.pathsep.join(dedupe_existing_dirs([*ld_dirs, *existing])) elif host.is_macos: