fix(studio/rocm): code review hardening pass
- main.py: numeric DLL sort (string sort picked rocm72 over rocm713); add basename() to regex; log warning on detection failure; log info when BNB_ROCM_VERSION is set (mirrors worker.py) - worker.py: explicit len-guard in _hip_ver_at_least() with warning logs instead of silent IndexError/ValueError swallow - hardware.py: isinstance(result, dict) guard before result.get() in _smi_query() to prevent AttributeError on non-dict backend returns - amd.py: round() before int() on parsed GPU IDs; log warning when truncation occurs (defensive against malformed amd-smi output) - setup.sh: quote --gcc-install-dir value in CMAKE_HIP_FLAGS so paths with spaces do not break the CMake argument - install.ps1, setup.ps1: apply colon-split + ToLower() to hipinfo gcnArchName match (consistent with each other and with setup.sh) - install.sh: tighten ROCm tag case patterns to explicit rocmX.Y|rocmX.Y.* to avoid unintended prefix matches
This commit is contained in:
LeoBorcherding
parent
e6cc98e75c
commit
06f28e4d03
8 changed files with 63 additions and 22 deletions
|
|
@ -1241,7 +1241,7 @@ shell.Run cmd, 0, False
|
|||
if ($LASTEXITCODE -eq 0 -and $hipOut -match "(?i)gcnArchName") {
|
||||
$HasROCm = $true
|
||||
if ($hipOut -match "(?im)^\s*gcnArchName\s*:\s*(\S+)") {
|
||||
$ROCmGfxArch = $Matches[1].Trim()
|
||||
$ROCmGfxArch = ($Matches[1] -split ':')[0].Trim().ToLower()
|
||||
$ROCmGpuLabel = "AMD ROCm ($ROCmGfxArch)"
|
||||
} else {
|
||||
$ROCmGpuLabel = "AMD ROCm"
|
||||
|
|
|
|||
16
install.sh
16
install.sh
|
|
@ -1647,14 +1647,14 @@ get_torch_index_url() {
|
|||
# PyTorch publishes major.minor URLs only (no patch level), so
|
||||
# rocm7.2.1 / rocm6.0.2 / etc. must normalise to rocm7.2 / rocm6.0.
|
||||
case "$_rocm_tag" in
|
||||
rocm6.0*) echo "$_base/rocm6.0" ;;
|
||||
rocm6.1*) echo "$_base/rocm6.1" ;;
|
||||
rocm6.2*) echo "$_base/rocm6.2" ;;
|
||||
rocm6.3*) echo "$_base/rocm6.3" ;;
|
||||
rocm6.4*) echo "$_base/rocm6.4" ;;
|
||||
rocm7.0*) echo "$_base/rocm7.0" ;;
|
||||
rocm7.1*) echo "$_base/rocm7.1" ;;
|
||||
rocm7.2*) echo "$_base/rocm7.2" ;;
|
||||
rocm6.0|rocm6.0.*) echo "$_base/rocm6.0" ;;
|
||||
rocm6.1|rocm6.1.*) echo "$_base/rocm6.1" ;;
|
||||
rocm6.2|rocm6.2.*) echo "$_base/rocm6.2" ;;
|
||||
rocm6.3|rocm6.3.*) echo "$_base/rocm6.3" ;;
|
||||
rocm6.4|rocm6.4.*) echo "$_base/rocm6.4" ;;
|
||||
rocm7.0|rocm7.0.*) echo "$_base/rocm7.0" ;;
|
||||
rocm7.1|rocm7.1.*) echo "$_base/rocm7.1" ;;
|
||||
rocm7.2|rocm7.2.*) echo "$_base/rocm7.2" ;;
|
||||
rocm6.*)
|
||||
# ROCm 6.5+ (no published PyTorch wheels): clip down
|
||||
# to the last supported 6.x wheel set.
|
||||
|
|
|
|||
|
|
@ -2149,8 +2149,24 @@ def run_training_process(
|
|||
return False
|
||||
try:
|
||||
_parts = [int(x) for x in str(_hip_str).split(".")[:2]]
|
||||
if len(_parts) < 2:
|
||||
logger.warning(
|
||||
"Windows ROCm: torch.version.hip %r has fewer than "
|
||||
"two components; cannot compare against %d.%d",
|
||||
_hip_str,
|
||||
major,
|
||||
minor,
|
||||
)
|
||||
return False
|
||||
return (_parts[0], _parts[1]) >= (major, minor)
|
||||
except (ValueError, IndexError):
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"Windows ROCm: could not parse torch.version.hip %r as "
|
||||
"a version number; assuming HIP < %d.%d",
|
||||
_hip_str,
|
||||
major,
|
||||
minor,
|
||||
)
|
||||
return False
|
||||
|
||||
# _grouped_mm HIP kernel was null on gfx1200 in ROCm ≤ 7.12,
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ if sys.platform == "win32":
|
|||
pass
|
||||
if _is_rocm_host and "BNB_ROCM_VERSION" not in os.environ:
|
||||
import glob as _glob
|
||||
import logging as _logging
|
||||
|
||||
_bnb_rocm_ver = None
|
||||
try:
|
||||
|
|
@ -99,14 +100,29 @@ if sys.platform == "win32":
|
|||
_dlls = _glob.glob(os.path.join(_pkg_dir, "libbitsandbytes_rocm*.dll"))
|
||||
import re as _re_bnb
|
||||
|
||||
for _dll in sorted(_dlls):
|
||||
_m = _re_bnb.search(r"libbitsandbytes_rocm(\d+)\.dll", _dll)
|
||||
def _bnb_ver_key(p: str) -> int:
|
||||
_km = _re_bnb.search(r"rocm(\d+)", os.path.basename(p))
|
||||
return int(_km.group(1)) if _km else -1
|
||||
|
||||
for _dll in sorted(_dlls, key=_bnb_ver_key, reverse=True):
|
||||
_m = _re_bnb.search(
|
||||
r"libbitsandbytes_rocm(\d+)\.dll", os.path.basename(_dll)
|
||||
)
|
||||
if _m:
|
||||
_bnb_rocm_ver = _m.group(1)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
os.environ["BNB_ROCM_VERSION"] = _bnb_rocm_ver or "72"
|
||||
except Exception as _e:
|
||||
_logging.getLogger(__name__).warning(
|
||||
"Windows ROCm: BNB DLL detection failed (%s); falling back to version '72'",
|
||||
_e,
|
||||
)
|
||||
_bnb_rocm_ver_final = _bnb_rocm_ver or "72"
|
||||
os.environ["BNB_ROCM_VERSION"] = _bnb_rocm_ver_final
|
||||
_logging.getLogger(__name__).info(
|
||||
"Windows ROCm: set BNB_ROCM_VERSION=%s "
|
||||
"(detected from installed BNB wheel; overrides torch.version.hip auto-detection)",
|
||||
_bnb_rocm_ver_final,
|
||||
)
|
||||
|
||||
# Ensure backend dir is on sys.path so _platform_compat is importable when
|
||||
# main.py is launched directly (e.g. `uvicorn main:app`).
|
||||
|
|
|
|||
|
|
@ -383,7 +383,7 @@ def get_visible_gpu_utilization(
|
|||
)
|
||||
parsed_id = _parse_numeric(raw_id)
|
||||
if parsed_id is None:
|
||||
logger.debug(
|
||||
logger.warning(
|
||||
"amd-smi GPU id %r could not be parsed; falling back to "
|
||||
"enumeration index %d",
|
||||
raw_id,
|
||||
|
|
@ -391,7 +391,15 @@ def get_visible_gpu_utilization(
|
|||
)
|
||||
idx = fallback_idx
|
||||
else:
|
||||
idx = int(parsed_id)
|
||||
rounded = round(parsed_id)
|
||||
if rounded != parsed_id:
|
||||
logger.warning(
|
||||
"amd-smi GPU id %r parsed as non-integer %r; truncating to %d",
|
||||
raw_id,
|
||||
parsed_id,
|
||||
rounded,
|
||||
)
|
||||
idx = int(rounded)
|
||||
if idx not in visible_set:
|
||||
continue
|
||||
metrics = _extract_gpu_metrics(gpu_data)
|
||||
|
|
|
|||
|
|
@ -467,7 +467,7 @@ def _smi_query(func_name: str, *args, **kwargs) -> Optional[Dict[str, Any]]:
|
|||
try:
|
||||
func = getattr(_backend, func_name)
|
||||
result = func(*args, **kwargs)
|
||||
if result.get("available"):
|
||||
if isinstance(result, dict) and result.get("available"):
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.warning("%s %s query failed: %s", backend_name, func_name, e)
|
||||
|
|
|
|||
|
|
@ -715,7 +715,7 @@ if (-not $HasNvidiaSmi) {
|
|||
if ($LASTEXITCODE -eq 0 -and $hipOut -match "(?i)gcnArchName") {
|
||||
$HasROCm = $true
|
||||
if ($hipOut -match "(?im)^\s*gcnArchName\s*:\s*(\S+)") {
|
||||
$script:ROCmGfxArch = $Matches[1].Trim()
|
||||
$script:ROCmGfxArch = ($Matches[1] -split ':')[0].Trim().ToLower()
|
||||
$ROCmGpuLabel = "AMD ROCm ($script:ROCmGfxArch)"
|
||||
} else {
|
||||
$ROCmGpuLabel = "AMD ROCm"
|
||||
|
|
|
|||
|
|
@ -1043,15 +1043,16 @@ else
|
|||
# runtime dir AND /usr/include/c++/<ver> headers, then pass it
|
||||
# to clang via --gcc-install-dir so HIP builds succeed.
|
||||
_GCC_INSTALL_DIR=""
|
||||
_GCC_MULTIARCH="$(gcc -print-multiarch 2>/dev/null || uname -m)-linux-gnu"
|
||||
for _gcc_ver in 14 13 12 11; do
|
||||
if [ -d "/usr/lib/gcc/x86_64-linux-gnu/$_gcc_ver/include" ] && \
|
||||
if [ -d "/usr/lib/gcc/$_GCC_MULTIARCH/$_gcc_ver/include" ] && \
|
||||
[ -d "/usr/include/c++/$_gcc_ver" ]; then
|
||||
_GCC_INSTALL_DIR="/usr/lib/gcc/x86_64-linux-gnu/$_gcc_ver"
|
||||
_GCC_INSTALL_DIR="/usr/lib/gcc/$_GCC_MULTIARCH/$_gcc_ver"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -n "$_GCC_INSTALL_DIR" ]; then
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_HIP_FLAGS=--gcc-install-dir=$_GCC_INSTALL_DIR"
|
||||
CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_HIP_FLAGS=--gcc-install-dir=\"$_GCC_INSTALL_DIR\""
|
||||
substep "ROCm HIP gcc install dir: $_GCC_INSTALL_DIR"
|
||||
fi
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue