diff --git a/install.ps1 b/install.ps1 index 1281103a35..b967eaf627 100644 --- a/install.ps1 +++ b/install.ps1 @@ -1371,9 +1371,7 @@ shell.Run cmd, 0, False $TorchIndexUrl = Get-TorchIndexUrl # ── AMD Windows ROCm wheel override ── - # AMD publishes direct torch wheels for Windows (cp312 only) at repo.radeon.com. - # When the HIP SDK is present and Python 3.12 is in use, swap in the AMD wheel - # URL and clear $TorchIndexUrl so the standard --index-url path is skipped. + # When the HIP SDK is present and Python 3.12, use repo.radeon.com direct wheels. $ROCmTorchWheelUrl = $null $ROCmTarballUrl = $null if ($HasROCm -and -not $SkipTorch) { @@ -1382,9 +1380,7 @@ shell.Run cmd, 0, False $amdWheelBase = if ($env:UNSLOTH_ROCM_WINDOWS_MIRROR) { $env:UNSLOTH_ROCM_WINDOWS_MIRROR.TrimEnd('/') } else { "https://repo.radeon.com/rocm/windows" } if ($ROCmVersion -and $ROCmVersion -match '^7\.2') { $amdRelBase = "$amdWheelBase/rocm-rel-7.2.1" - # rocm tarball (14 KB) provides the 'rocm_sdk' Python namespace that - # torch/_rocm_init.py imports at startup. - $ROCmTarballUrl = "$amdRelBase/rocm-7.2.1.tar.gz" + $ROCmTarballUrl = "$amdRelBase/rocm-7.2.1.tar.gz" # rocm_sdk namespace $ROCmAllWheelUrls = @( "$amdRelBase/rocm_sdk_core-7.2.1-py3-none-win_amd64.whl", "$amdRelBase/rocm_sdk_devel-7.2.1-py3-none-win_amd64.whl", @@ -1397,9 +1393,7 @@ shell.Run cmd, 0, False $TorchIndexUrl = $null } elseif ($ROCmVersion -and $ROCmVersion -match '^7\.1') { $amdRelBase = "$amdWheelBase/rocm-rel-7.1.1" - # rocm tarball (14 KB) provides the 'rocm_sdk' Python namespace that - # torch/_rocm_init.py imports at startup. - $ROCmTarballUrl = "$amdRelBase/rocm-0.1.dev0.tar.gz" + $ROCmTarballUrl = "$amdRelBase/rocm-0.1.dev0.tar.gz" # rocm_sdk namespace $ROCmAllWheelUrls = @( "$amdRelBase/rocm_sdk_core-0.1.dev0-py3-none-win_amd64.whl", "$amdRelBase/rocm_sdk_libraries_custom-0.1.dev0-py3-none-win_amd64.whl", @@ -1423,7 +1417,11 @@ shell.Run cmd, 0, False } } - $TorchIndexFamily = Get-TauriTorchIndexFamily $(if ($ROCmTorchWheelUrl) { "rocm7.2" } else { $TorchIndexUrl }) + $TorchIndexFamily = Get-TauriTorchIndexFamily $( + if ($ROCmTorchWheelUrl) { + if ($ROCmVersion -match '^7\.1') { "rocm7.1" } else { "rocm7.2" } + } else { $TorchIndexUrl } + ) $GpuBranch = Get-TauriGpuBranch $TorchIndexFamily Write-TauriDiag -GpuBranch $GpuBranch -TorchIndexFamily $TorchIndexFamily -PythonVersionForDiag $DetectedPython.Version @@ -1512,16 +1510,13 @@ shell.Run cmd, 0, False } elseif ($ROCmTorchWheelUrl) { Write-TauriLog "STEP" "Installing PyTorch (AMD ROCm Windows)" substep "installing PyTorch (AMD ROCm $ROCmVersion)..." - # Install the rocm namespace tarball first (provides the 'rocm_sdk' - # Python package that torch/_rocm_init.py imports at startup). + # rocm_sdk namespace tarball (torch/_rocm_init.py imports it at startup) if ($ROCmTarballUrl) { $tarballExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --no-deps $ROCmTarballUrl } if ($tarballExit -ne 0) { Write-Host "[WARN] ROCm namespace tarball install failed (exit $tarballExit) -- continuing" -ForegroundColor Yellow } } - # Install remaining SDK + torch wheels. @array splatting inside a - # scriptblock works in PS 5.1 because & $Command runs in-scope. $torchInstallExit = Invoke-InstallCommand { uv pip install --python $VenvPython --force-reinstall --no-deps @ROCmAllWheelUrls } if ($torchInstallExit -ne 0) { Write-Host "[ERROR] Failed to install AMD ROCm PyTorch (exit code $torchInstallExit)" -ForegroundColor Red diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index a4751d2e69..ba60aa802c 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -1085,18 +1085,12 @@ def run_training_process( ) # ── 1d. Ensure torch.distributed is importable before ML libs load ── - # The Windows ROCm wheel ships without torch._C._distributed_c10d (the C - # backend for the distributed package). This causes two distinct failure - # modes that must both be handled: - # - # (a) `import torch.distributed` raises ImportError immediately, OR - # (b) the import SUCCEEDS (the symbol is lazily resolved) but the first - # actual call by transformers/trl triggers the missing-module error. - # - # Strategy: on Windows, unconditionally pre-stub torch._C._distributed_c10d - # in sys.modules AND as an attribute on the torch._C extension module BEFORE - # attempting the import. That covers both (a) and (b). Then do the import - # and backfill any missing helper attributes on torch.distributed itself. + # The Windows ROCm wheel ships without torch._C._distributed_c10d. + # Two failure modes: (a) ImportError on `import torch.distributed`, or + # (b) the import succeeds (lazy load) but the first call by trl/transformers + # crashes. Pre-stubbing before the import covers both. + # Guard with `not in sys.modules` so we never overwrite a real CUDA/NVIDIA + # implementation that was already loaded. import types as _types _td_stubs = { @@ -1109,19 +1103,17 @@ def run_training_process( } if sys.platform == "win32": - # Pre-stub the missing C extension so both the module-import path and - # the attribute-access path (`from torch._C import _distributed_c10d`) - # return a harmless no-op object instead of raising ImportError. _c10d_key = "torch._C._distributed_c10d" - _c10d_stub = _types.ModuleType(_c10d_key) - sys.modules[_c10d_key] = _c10d_stub - try: - import torch._C as _torch_C_mod # C ext — always importable + if _c10d_key not in sys.modules: + _c10d_stub = _types.ModuleType(_c10d_key) + sys.modules[_c10d_key] = _c10d_stub + try: + import torch._C as _torch_C_mod # C ext — always importable - if not hasattr(_torch_C_mod, "_distributed_c10d"): - _torch_C_mod._distributed_c10d = _c10d_stub - except Exception: - pass + if not hasattr(_torch_C_mod, "_distributed_c10d"): + _torch_C_mod._distributed_c10d = _c10d_stub + except Exception: + pass try: import torch.distributed as _td @@ -1134,7 +1126,6 @@ def run_training_process( for _name, _stub in _td_stubs.items(): setattr(_td_mock, _name, _stub) sys.modules["torch.distributed"] = _td_mock - # Ensure C extension stub survives (may have been wiped by a failed import) if "torch._C._distributed_c10d" not in sys.modules: sys.modules["torch._C._distributed_c10d"] = _types.ModuleType( "torch._C._distributed_c10d" diff --git a/studio/backend/main.py b/studio/backend/main.py index 812ddd45b9..46f5b97a86 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -13,13 +13,9 @@ from pathlib import Path as _Path os.environ["PYTHONWARNINGS"] = "ignore" # ── Windows AMD ROCm DLL injection ────────────────────────────────────────── -# On Windows, Python 3.8+ uses a secure DLL search that ignores PATH for -# extension modules. torch's HIP backend (amdhip64.dll etc.) won't be found -# even if F:\ROCm\...\bin is in PATH unless we explicitly register the -# directory with os.add_dll_directory(). Do this before any torch import. +# Python 3.8+ ignores PATH for extension modules; register ROCm bin dirs with +# os.add_dll_directory() so amdhip64.dll etc. are found before any torch import. if sys.platform == "win32": - import ctypes as _ctypes - def _add_rocm_dll_dirs() -> None: candidates = [] # 1. HIP_PATH / ROCM_PATH -- set by the AMD HIP SDK installer @@ -48,7 +44,7 @@ if sys.platform == "win32": pass _add_rocm_dll_dirs() - del _add_rocm_dll_dirs, _ctypes + del _add_rocm_dll_dirs # Ensure backend dir is on sys.path so _platform_compat is importable when # main.py is launched directly (e.g. `uvicorn main:app`). diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py index 1b990b6adf..3593d9d677 100644 --- a/studio/backend/utils/hardware/hardware.py +++ b/studio/backend/utils/hardware/hardware.py @@ -516,12 +516,7 @@ def get_gpu_utilization() -> Dict[str, Any]: if result is not None: result["backend"] = _backend_label(device) if IS_ROCM: - # Mirror the unified-memory reconciliation done in the - # visible-GPU path. amd-smi on AMD iGPUs (Strix Halo etc.) - # reports only the dedicated VRAM slice; torch.mem_get_info - # sees the full GTT pool. Without this the /api/train/hardware - # endpoint and the live GPU monitor still display the wrong - # VRAM total even after auto-selection has been corrected. + # Fix unified-memory VRAM on AMD iGPUs (Strix Halo etc.) _reconcile_primary_rocm_unified_memory( result, _get_parent_visible_gpu_spec() ) @@ -621,14 +616,11 @@ def _apply_unified_memory_correction( def _reconcile_rocm_unified_memory( utilization: Dict[str, Any], device_indices: list[int] ) -> None: - """Cross-check amd-smi VRAM data against torch mem_get_info for ROCm. + """Fix amd-smi VRAM for ROCm unified-memory GPUs (e.g. Strix Halo). - On AMD iGPUs with unified/shared memory (e.g. Strix Halo / Radeon 8060S), - amd-smi reports only the dedicated VRAM slice (typically 512 MB) in its - metric output, while torch.cuda.mem_get_info() surfaces the full GTT / - unified pool (~128 GB). When torch reports a larger total than amd-smi, - replace the per-device VRAM fields so auto_select_gpu_ids sees the real - usable memory instead of the tiny dedicated slice. + amd-smi reports only the dedicated slice (~512 MB); torch sees the full + GTT pool (~128 GB). When torch total > smi total, overwrite per-device + VRAM fields so GPU selection uses the real available memory. """ torch_devices = _torch_get_per_device_info(device_indices) if not torch_devices: @@ -644,14 +636,7 @@ def _reconcile_rocm_unified_memory( def _reconcile_primary_rocm_unified_memory( utilization: Dict[str, Any], parent_visible_spec: Dict[str, Any] ) -> None: - """Primary-GPU variant of the unified-memory reconciliation. - - ``get_primary_gpu_utilization`` returns a flat metrics dict (no nested - ``devices`` list) for the first visible AMD GPU. Run the same correction - against torch.mem_get_info for that single device so the live training - hardware endpoint and the GPU monitor surface the real unified-memory - pool on Strix Halo and similar iGPUs. - """ + """Same fix as _reconcile_rocm_unified_memory for the flat primary-GPU dict.""" numeric_ids = parent_visible_spec.get("numeric_ids") if numeric_ids: primary_idx = [int(numeric_ids[0])] @@ -678,10 +663,7 @@ def get_visible_gpu_utilization() -> Dict[str, Any]: if result is not None: result["backend"] = _backend_label(device) if IS_ROCM: - # amd-smi on iGPUs with unified memory (e.g. Strix Halo) - # reports only the dedicated VRAM slice; torch mem_get_info - # sees the full unified pool. Reconcile so downstream GPU - # selection uses the real available memory. + # Fix unified-memory VRAM on AMD iGPUs (Strix Halo etc.) _reconcile_rocm_unified_memory( result, parent_visible_spec["numeric_ids"] ) diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index c6bbfda385..0eb1a81d40 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -78,8 +78,6 @@ _ROCM_WINDOWS_WHEEL_BASE = ( or "https://repo.radeon.com/rocm/windows" ).rstrip("/") # Maps (major, minor) → (release_folder, [wheel_filename, ...]) -# Includes rocm_sdk_core and rocm_sdk_libraries_custom because the torch -# wheels declare them as hard dependencies (rocm[libraries]==). _ROCM_WINDOWS_RELEASES: dict[tuple[int, int], tuple[str, list[str]]] = { (7, 2): ( "rocm-rel-7.2.1", @@ -334,9 +332,7 @@ def _ensure_rocm_torch() -> None: Uses pip_install() to respect uv, constraints, and --python targeting. """ global _rocm_windows_torch_installed - # setup.ps1 sets this env var when it successfully installs AMD wheels - # before calling install_python_stack.py, so we can skip the subprocess - # probe and avoid reinstalling what was just installed. + # setup.ps1 sets this when it already installed AMD wheels; skip the probe. if os.environ.get("UNSLOTH_ROCM_TORCH_INSTALLED") == "1": _rocm_windows_torch_installed = True return diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 81aa66c999..ec8668b6a4 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -657,8 +657,8 @@ class TestEnsureRocmTorch: @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False) @patch.object(stack_mod, "_has_rocm_gpu", return_value = True) @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 2)) - def test_rocm_72_selects_71_tag(self, mock_ver, mock_gpu, mock_nvidia, mock_pip): - """ROCm 7.2 should select rocm7.1 tag (capped, not in mapping).""" + def test_rocm_72_selects_72_tag(self, mock_ver, mock_gpu, mock_nvidia, mock_pip): + """ROCm 7.2 should select rocm7.2 tag (now in mapping with torch 2.11.0).""" mock_probe = MagicMock() mock_probe.returncode = 0 mock_probe.stdout = b"\n" @@ -666,7 +666,7 @@ class TestEnsureRocmTorch: with patch("subprocess.run", return_value = mock_probe): _ensure_rocm_torch() torch_call = mock_pip.call_args_list[0] - assert "rocm7.1" in str(torch_call) + assert "rocm7.2" in str(torch_call) @patch.object(stack_mod, "pip_install_try", return_value = True) @patch.object(stack_mod, "pip_install") @@ -711,9 +711,10 @@ class TestRocmTorchIndex: keys = list(_ROCM_TORCH_INDEX.keys()) assert keys == sorted(keys, reverse = True) - def test_rocm_72_not_in_mapping(self): - """ROCm 7.2 should NOT be in the active mapping (torch 2.11.0 exceeds bound).""" - assert (7, 2) not in _ROCM_TORCH_INDEX + def test_rocm_72_in_mapping(self): + """ROCm 7.2 should be in the active mapping (torch 2.11.0 now supported).""" + assert (7, 2) in _ROCM_TORCH_INDEX + assert _ROCM_TORCH_INDEX[(7, 2)] == "rocm7.2" def test_rocm_71_maps_correctly(self): assert _ROCM_TORCH_INDEX[(7, 1)] == "rocm7.1" @@ -731,7 +732,7 @@ class TestRocmTorchIndex: assert "radeon" not in tag def test_newer_rocm_selects_best_match(self): - """ROCm 7.2 (not in map) should select rocm7.1 via >= comparison.""" + """ROCm 7.2 (now in map) should select rocm7.2 directly.""" ver = (7, 2) tag = next( ( @@ -741,7 +742,7 @@ class TestRocmTorchIndex: ), None, ) - assert tag == "rocm7.1" + assert tag == "rocm7.2" def test_rocm_64_selects_64(self): ver = (6, 4) @@ -927,15 +928,16 @@ class TestInstallShStructure: source = sh_path.read_text() assert "ROCm" in source - def test_rocm72_capped_to_71(self): - """ROCm 7.2+ should fall back to rocm7.1 index.""" + def test_rocm72_supported_future_capped(self): + """ROCm 7.2 should pass through directly; 7.3+ falls back to rocm7.2.""" sh_path = PACKAGE_ROOT / "install.sh" source = sh_path.read_text() - assert 'echo "$_base/rocm7.1"' in source # fallback for unknown versions + assert 'echo "$_base/rocm7.2"' in source # fallback for unknown future versions # Allowlisted versions should pass through directly assert "rocm6.*" in source assert "rocm7.0" in source assert "rocm7.1" in source + assert "rocm7.2" in source def test_rocm_tag_validation_guard_exists(self): """install.sh should validate _rocm_tag with a case guard."""