diff --git a/install.ps1 b/install.ps1 index c36b60f3b8..0b0f5478af 100644 --- a/install.ps1 +++ b/install.ps1 @@ -61,7 +61,11 @@ function Install-UnslothStudio { function Get-TauriGpuBranch { param([string]$TorchIndexFamily) if ($SkipTorch) { return "no_torch" } - if ($TorchIndexFamily -like "cu*") { return "cuda" } + # Require digits after "cu" (cu118/cu128/...) so an odd mirror leaf like + # "custom"/"current" is not mis-branded CUDA. $TorchIndexFamily is already + # normalised by Get-TauriTorchIndexFamily today, but keep the guard narrow + # to match the ^cu[0-9] rule in setup.ps1 / install_python_stack.py. + if ($TorchIndexFamily -match '^cu[0-9]') { return "cuda" } if ($TorchIndexFamily -like "rocm*") { return "rocm" } if ($TorchIndexFamily -eq "cpu") { return "cpu" } return "unknown" diff --git a/install.sh b/install.sh index 2e6db6c12a..0120197bbd 100755 --- a/install.sh +++ b/install.sh @@ -334,7 +334,11 @@ _tauri_gpu_branch() { return fi case "$_diag_family" in - cu*) echo "cuda" ;; + # Require a digit after cu (cu118/cu128/...) so an odd leaf like custom / + # current is not branded CUDA -- matches the ^cu[0-9] rule in setup.ps1 / + # install_python_stack.py. $_diag_family is already normalised by + # _tauri_torch_index_family, but keep the guard narrow for parity. + cu[0-9]*) echo "cuda" ;; rocm*) if [ "$_diag_radeon" = true ]; then echo "rocm_radeon" diff --git a/studio/install_python_stack.py b/studio/install_python_stack.py index 346e5659d1..2ac0ad042c 100644 --- a/studio/install_python_stack.py +++ b/studio/install_python_stack.py @@ -1091,34 +1091,62 @@ def _rocm_pin_family_mismatch(pin_url: str, installed_ver: str) -> bool: """True when an explicit ROCm pin names a different ROCm family than the already-installed ROCm torch, so the pin needs a reinstall to be applied. - Mirrors setup.ps1's stale-venv ROCm comparison: - - both +rocmX.Y versions readable -> compare them exactly - - gfx pin, or an unreadable installed version -> compare on the torch 2.11 - line (gfx*/rocm>=7.2 serve 2.11+, older ROCm does not) + Mirrors setup.ps1's stale-venv ROCm comparison. The pin leaf classifies into + three cases, matching the install-spec path in _ensure_rocm_torch: + * rocmX.Y leaf -> compare the pinned rocm version to the installed one + exactly when both are readable; else fall back to the torch 2.11 line + (rocm>=7.2 serves 2.11, older rocm does not). + * gfx leaf in _ROCM_GFX_TORCH211_LEAVES (gfx120x-all/gfx1151/gfx1150) -> + the install path pulls AMD's per-arch wheel (tagged with a THREE-part + +rocmA.B.C local version, e.g. 2.11.0+rocm7.13.0). A generic pytorch.org + rocm wheel (two-part +rocmA.B, e.g. +rocm7.2) or any pre-2.11 build is a + mismatch even when both are torch 2.11 -- the user asked for the per-arch + index. An already-installed per-arch wheel (three-part tag) is NOT a + mismatch, so a satisfied gfx pin does not reinstall-loop. + * gfx leaf NOT in the 2.11 allowlist (gfx110X-all/gfx90a/gfx908) -> the + install path uses the default <2.11 specs, so a correct 2.10+rocm wheel + must NOT be flagged. Mismatch only when the installed torch is 2.11+. A pin that resolves to the same family as what is installed is NOT a mismatch, so a correct ROCm venv is never needlessly reinstalled. Pure function. """ leaf = pin_url.rstrip("/").rsplit("/", 1)[-1].lower() - # Pinned ROCm version (from a rocmX.Y leaf) and whether the pin serves 2.11+. + # Pinned ROCm version (from a rocmX.Y leaf). _pin_rocm = re.match(r"^rocm(\d+)\.(\d+)", leaf) _pin_ver = (int(_pin_rocm.group(1)), int(_pin_rocm.group(2))) if _pin_rocm else None - if leaf.startswith("gfx"): - _pin_is_211 = True - elif _pin_ver is not None: - _pin_is_211 = _pin_ver >= (7, 2) - else: - _pin_is_211 = False - # Installed ROCm version (+rocmX.Y) and whether the installed torch is 2.11+. + # Installed ROCm version (+rocmX.Y) and whether the installed wheel carries a + # THREE-part local version (+rocmA.B.C) -- the AMD per-arch signature that + # distinguishes a repo.amd.com/gfx* wheel from a two-part pytorch.org one. _inst_rocm = re.search(r"\+rocm(\d+)\.(\d+)", installed_ver) _inst_ver = (int(_inst_rocm.group(1)), int(_inst_rocm.group(2))) if _inst_rocm else None + _inst_is_perarch = re.search(r"\+rocm\d+\.\d+\.\d+", installed_ver) is not None + # Whether the installed torch RELEASE (before "+") is 2.11+. _inst_rel = re.match(r"^(\d+)\.(\d+)", installed_ver) _inst_is_211 = ( (int(_inst_rel.group(1)), int(_inst_rel.group(2))) >= (2, 11) if _inst_rel else False ) + + if leaf.startswith("gfx"): + # gfx per-arch pin: only the _grouped_mm-bug arches (the 2.11 allowlist) + # pull the AMD per-arch wheel; other gfx leaves stay on the default + # <2.11 specs (see _ROCM_TORCH_PKG_SPECS selection below). + if leaf in _ROCM_GFX_TORCH211_LEAVES: + # Expect the AMD per-arch wheel (three-part +rocmA.B.C, torch 2.11+). + # A satisfied per-arch install is NOT a mismatch (no reinstall loop); + # a generic rocm wheel or any pre-2.11 build IS a mismatch even at 2.11. + return not (_inst_is_211 and _inst_is_perarch) + # Non-2.11 gfx leaf: install path uses default <2.11 specs, so a correct + # <2.11 wheel must stay. Mismatch only when the installed torch is 2.11+. + return _inst_is_211 + + # rocmX.Y pin. + _pin_is_211 = _pin_ver >= (7, 2) if _pin_ver is not None else False if _pin_ver is not None and _inst_ver is not None: - # Both ROCm versions readable: exact comparison. + # Both ROCm versions readable: exact (major, minor) comparison. A generic + # rocm7.2 pin over the AMD per-arch (+rocm7.13.x) wheel compares (7, 2) vs + # (7, 13) -> mismatch, which correctly reinstalls the generic wheel the + # user pinned instead of leaving the per-arch one in place. return _pin_ver != _inst_ver - # gfx pin or unreadable version: compare on the torch 2.11 line. + # rocm pin with an unreadable installed version: compare on the torch 2.11 line. return _pin_is_211 != _inst_is_211 @@ -1524,9 +1552,12 @@ def _ensure_rocm_torch() -> None: # capture the installed ROCm build tag so a pin mismatch can be detected. # Do NOT skip for CUDA-only builds: they are unusable on AMD-only hosts # (the NVIDIA check above already handled mixed AMD+NVIDIA setups). - # Line 1: the HIP presence marker (HIP version, "rocm" sentinel, or ""). - # Line 2: the installed wheel version string (e.g. "2.10.0+rocm6.4"), used to - # compare the installed ROCm family against an explicit pin below. + # Emit ONE "|" line (mirrors _ensure_cuda_torch) so the + # parse is positional and robust: the HIP marker is the field before "|" (HIP + # version, "rocm" sentinel, or empty for CPU/CUDA torch), the installed wheel + # version (e.g. "2.10.0+rocm6.4") is the field after it. Do NOT filter empty + # LINES and take slot 0 -- for CPU/CUDA torch the marker field IS empty, and + # dropping it would shift the version into slot 0 and wrongly flag has_hip_torch. try: probe = subprocess.run( [ @@ -1536,11 +1567,11 @@ def _ensure_rocm_torch() -> None: "import torch; " "hip=getattr(torch.version,'hip','') or ''; " "ver=getattr(torch,'__version__','').lower(); " - # Print the HIP version when present (back-compat), else a - # "rocm" sentinel when only torch.__version__ flags ROCm - # (AMD SDK / Radeon wheels). Empty string = CPU/CUDA. - "print(hip if hip else ('rocm' if 'rocm' in ver else '')); " - "print(ver)" + # HIP version when present (back-compat), else a "rocm" + # sentinel when only torch.__version__ flags ROCm (AMD SDK / + # Radeon wheels). Empty marker before "|" = CPU/CUDA torch. + "marker=hip if hip else ('rocm' if 'rocm' in ver else ''); " + "print(marker + '|' + ver)" ), ], stdout = subprocess.PIPE, @@ -1549,13 +1580,22 @@ def _ensure_rocm_torch() -> None: ) except (OSError, subprocess.TimeoutExpired): probe = None - _probe_lines = ( + # Take the last non-empty stdout line so stray sitecustomize / import-hook + # output cannot mask the marker; then split positionally on the FIRST "|" -- + # the empty HIP-marker field for CPU/CUDA torch is preserved (has_hip_torch + # is driven by that field, not by "first non-empty line"). + _marker_lines = ( [ln.strip() for ln in probe.stdout.decode(errors = "replace").splitlines() if ln.strip()] if (probe is not None and probe.returncode == 0) else [] ) - has_hip_torch = bool(_probe_lines) and _probe_lines[0] != "" - _installed_torch_ver = _probe_lines[1] if len(_probe_lines) > 1 else "" + _hip_marker, _sep, _installed_torch_ver = ( + _marker_lines[-1].partition("|") if _marker_lines else ("", "", "") + ) + # A "|"-delimited marker line is required: without the separator the probe + # output is unrecognised (old torch, injected noise), so treat HIP as absent + # and fall through to a reinstall rather than trusting an ambiguous string. + has_hip_torch = bool(_sep) and _hip_marker != "" # An explicit ROCm pin whose family differs from the already-installed ROCm # torch must reinstall, mirroring _ensure_cuda_torch (installed cuXXX != pin). diff --git a/studio/setup.ps1 b/studio/setup.ps1 index abab5d0ef8..a966b82afc 100644 --- a/studio/setup.ps1 +++ b/studio/setup.ps1 @@ -425,6 +425,81 @@ function Get-TorchIndexLeaf { return ($Url.TrimEnd('/') -split '/')[-1].ToLowerInvariant() } +# The AMD per-arch index leaves that need the torch 2.11 floor (the _grouped_mm +# null-ptr bug lives in the <2.11 wheels for these arches). MUST match the +# $_pinGfx211 allowlist in the install-spec path below (and install.ps1 / +# install_python_stack.py). Other per-arch leaves (gfx110X-all/gfx90a/gfx908) +# publish <2.11 wheels and stay on default specs, so a pin to one of those must +# NOT be judged stale against the 2.11 line. +function Test-RocmGfx211Leaf { + param([string]$Leaf) + return @('gfx120x-all', 'gfx1151', 'gfx1150') -contains $Leaf +} + +# True only for a real CUDA wheel-family leaf: "cu" followed by digits (cu118, +# cu126, cu128, cu130, ...). Mirrors install_python_stack.py::_is_cuda_family_leaf. +# A bare -like 'cu*' wrongly matches arbitrary mirror leaves like "custom" / +# "current", which would set an expected tag no installed flavor can equal and +# rebuild the venv on every run instead of trusting the unknown custom index. +function Test-CudaFamilyLeaf { + param([string]$Leaf) + if ([string]::IsNullOrWhiteSpace($Leaf)) { return $false } + return $Leaf -match '^cu[0-9]' +} + +# Stale-venv ROCm comparison for a pinned gfx*/rocm* index. Returns a hashtable +# @{ Expected = ; Installed = } so the caller rebuilds when they +# differ. Mirrors install_python_stack.py::_rocm_pin_family_mismatch: +# * rocmX.Y pin -> compare exact rocm versions when both readable, else the +# torch 2.11 line (rocm>=7.2 serves 2.11, older rocm does not). +# * gfx pin in the 2.11 allowlist -> expect AMD's per-arch wheel (three-part +# +rocmA.B.C local version). A generic (two-part +rocmA.B) or pre-2.11 wheel +# is stale; an already-installed per-arch wheel is NOT (no rebuild loop). +# * gfx pin NOT in the allowlist (gfx110X-all/gfx90a/gfx908) -> default <2.11 +# specs, so a 2.10+rocm wheel is correct; only a 2.11+ build is stale. +function Get-RocmPinStaleTags { + param([string]$PinLeaf, [string]$TorchVersion) + $_pinRocm = [regex]::Match($PinLeaf, '^rocm(\d+)\.(\d+)') + $_pinVer = if ($_pinRocm.Success) { "$($_pinRocm.Groups[1].Value).$($_pinRocm.Groups[2].Value)" } else { $null } + # Installed rocm version and whether the wheel is a per-arch (three-part) build. + $_instRocm = [regex]::Match($TorchVersion, '\+rocm(\d+)\.(\d+)') + $_instVer = if ($_instRocm.Success) { "$($_instRocm.Groups[1].Value).$($_instRocm.Groups[2].Value)" } else { $null } + $_instPerArch = [regex]::IsMatch($TorchVersion, '\+rocm\d+\.\d+\.\d+') + $_instRel = [regex]::Match($TorchVersion, '^(\d+)\.(\d+)') + $_instIs211 = $false + if ($_instRel.Success) { + $_instIs211 = ([int]$_instRel.Groups[1].Value -gt 2) -or ([int]$_instRel.Groups[1].Value -eq 2 -and [int]$_instRel.Groups[2].Value -ge 11) + } + + if ($PinLeaf -like 'gfx*') { + if (Test-RocmGfx211Leaf $PinLeaf) { + # Expect the AMD per-arch (three-part) 2.11 wheel. Satisfied only when + # BOTH a 2.11 release AND a three-part rocm tag are installed. + $installed = if ($_instIs211 -and $_instPerArch) { "rocm-perarch(torch>=2.11)" } else { "rocm-generic-or-old" } + return @{ Expected = "rocm-perarch(torch>=2.11)"; Installed = $installed } + } + # Non-2.11 gfx leaf: default <2.11 spec. Stale only when the build is 2.11+. + return @{ + Expected = "rocm(torch<2.11)" + Installed = if ($_instIs211) { "rocm(torch>=2.11)" } else { "rocm(torch<2.11)" } + } + } + + # rocmX.Y pin. + if ($_pinVer -and $_instVer) { + # Both rocm versions readable: exact comparison. + return @{ Expected = "rocm$_pinVer"; Installed = "rocm$_instVer" } + } + $_pinNeeds211 = $false + if ($_pinRocm.Success) { + $_pinNeeds211 = ([int]$_pinRocm.Groups[1].Value -gt 7) -or ([int]$_pinRocm.Groups[1].Value -eq 7 -and [int]$_pinRocm.Groups[2].Value -ge 2) + } + return @{ + Expected = if ($_pinNeeds211) { "rocm(torch>=2.11)" } else { "rocm(torch<2.11)" } + Installed = if ($_instIs211) { "rocm(torch>=2.11)" } else { "rocm(torch<2.11)" } + } +} + # VS generator -> MSBuild BuildCustomizations dir; toolset tracks the VS major # (18->v180, 17->v170), defaulting to v170 when unparseable. function Get-VcBuildCustomizationsDir { @@ -2573,35 +2648,20 @@ if ((Test-Path -LiteralPath $VenvDir -PathType Container) -and -not $NoTorchMode # Do NOT collapse a pinned ROCm/gfx leaf to a generic "rocm": that # would match any installed +rocm wheel and mask a pin change from # one ROCm family to another (e.g. rocm6.4 -> gfx1151, or rocm6.4 - # -> rocm6.3), leaving the requested index unapplied. Compare what - # the wheel tag exposes -- the ROCm version (+rocmX.Y) and whether - # the index serves the torch 2.11 line (gfx* / rocm>=7.2 do; older - # rocm does not). gfx pins carry no rocm version in the leaf, so - # they compare on the 2.11 line only. - $_pinNeeds211 = $false - $_pinRocmVer = $null - if ($_pinLeaf -like 'gfx*') { - $_pinNeeds211 = $true - } elseif ($_pinLeaf -match '^rocm(\d+)\.(\d+)') { - $_pinRocmVer = "$($Matches[1]).$($Matches[2])" - $_pinNeeds211 = ([int]$Matches[1] -gt 7) -or ([int]$Matches[1] -eq 7 -and [int]$Matches[2] -ge 2) - } - $_instRocmVer = $null - if ($torchVer -match '\+rocm(\d+)\.(\d+)') { $_instRocmVer = "$($Matches[1]).$($Matches[2])" } - $_instIs211 = $false - if ($torchVer -match '^(\d+)\.(\d+)') { - $_instIs211 = ([int]$Matches[1] -gt 2) -or ([int]$Matches[1] -eq 2 -and [int]$Matches[2] -ge 11) - } - if ($_pinRocmVer -and $_instRocmVer) { - # Both ROCm versions readable: compare them exactly. - $expectedTorchTag = "rocm$_pinRocmVer" - $installedTorchTag = "rocm$_instRocmVer" - } else { - # gfx pin or unreadable version: compare on the torch 2.11 line. - $expectedTorchTag = if ($_pinNeeds211) { "rocm(torch>=2.11)" } else { "rocm(torch<2.11)" } - $installedTorchTag = if ($_instIs211) { "rocm(torch>=2.11)" } else { "rocm(torch<2.11)" } - } - } elseif ($_pinLeaf -like 'cu*' -or $_pinLeaf -eq 'cpu') { + # -> rocm6.3), leaving the requested index unapplied. Get-RocmPinStaleTags + # compares what the wheel tag exposes and uses the SAME 2.11 allowlist + # (Test-RocmGfx211Leaf) as the install-spec path, so a gfx110X-all / + # gfx90a / gfx908 pin on a valid <2.11 wheel is NOT judged stale, and a + # 2.11-allowlist gfx pin over a generic (two-part +rocm) wheel IS. + $_rocmTags = Get-RocmPinStaleTags -PinLeaf $_pinLeaf -TorchVersion $torchVer + $expectedTorchTag = $_rocmTags.Expected + $installedTorchTag = $_rocmTags.Installed + } elseif ((Test-CudaFamilyLeaf $_pinLeaf) -or $_pinLeaf -eq 'cpu') { + # Require digits after "cu" (cu118/cu128/...) so a mirror leaf like + # /custom or /current is NOT treated as a CUDA flavor -- otherwise the + # expected tag would be an arbitrary word no installed flavor equals, + # rebuilding the venv every run. Such leaves fall through to the + # trust-unknown-index branch below. $expectedTorchTag = $_pinLeaf } else { # Custom index whose final segment is not a torch flavor (e.g. a @@ -2928,8 +2988,9 @@ if ($TorchIndexPinned -and -not $ROCmIndexUrl -and $PinnedTorchIndexUrl) { # floor here (gfx120X-all, gfx1151, gfx1150 -- the _grouped_mm bug arches). # Other per-arch indexes (gfx110X-all, gfx90a, gfx908) publish <2.11 wheels # and the automatic path leaves them bare, so an override to one of those - # must NOT force a 2.11 floor the normal path intentionally avoids. - $_pinGfx211 = @('gfx120x-all', 'gfx1151', 'gfx1150') -contains $_pinLeaf + # must NOT force a 2.11 floor the normal path intentionally avoids. Reuse + # Test-RocmGfx211Leaf so this allowlist and the stale-venv check never diverge. + $_pinGfx211 = Test-RocmGfx211Leaf $_pinLeaf if ($_pinGfx211 -or $_pinRocm211) { $ROCmIndexUrl = $PinnedTorchIndexUrl $ROCmTorchSpec = "torch>=2.11.0,<2.12.0" diff --git a/tests/python/test_cross_platform_parity.py b/tests/python/test_cross_platform_parity.py index c04e7e4b4d..6dd7307c7f 100644 --- a/tests/python/test_cross_platform_parity.py +++ b/tests/python/test_cross_platform_parity.py @@ -255,3 +255,81 @@ class TestTorchIndexOverrideParity: "setup.ps1 stale check should restrict the ROCm expected-tag to the " "supported gfx wheel arches" ) + + +class TestGfx211AllowlistParity: + """The gfx per-arch leaves that carry the torch 2.11 floor (gfx120X-all / + gfx1151 / gfx1150) must be the SAME set in every installer AND in each + installer's stale/mismatch check. When these diverged, a pinned gfx110X-all / + gfx90a / gfx908 wheel (which stays <2.11) was force-reinstalled every update.""" + + EXPECTED = {"gfx120x-all", "gfx1151", "gfx1150"} + + def test_install_sh_allowlist(self): + text = INSTALL_SH.read_text(encoding = "utf-8").lower() + # install.sh: the TORCH_CONSTRAINT case (rocm7.2|gfx120x-all|gfx1151|gfx1150). + m = re.search(r"rocm7\.2\|gfx120x-all\|gfx1151\|gfx1150", text) + assert m, "install.sh gfx-2.11 allowlist case not found / changed" + + def test_install_ps1_allowlist(self): + text = INSTALL_PS1.read_text(encoding = "utf-8").lower() + m = re.search(r"@\('gfx120x-all',\s*'gfx1151',\s*'gfx1150'\)", text) + assert m, "install.ps1 $_pinGfx211 allowlist not found / changed" + + def test_setup_ps1_defines_single_allowlist_helper(self): + # setup.ps1 must define the allowlist once (Test-RocmGfx211Leaf) and the + # install-spec path must reuse it, so the stale check and install spec can + # never disagree again. + text = SETUP_PS1.read_text(encoding = "utf-8") + assert "function Test-RocmGfx211Leaf" in text, ( + "setup.ps1 should define a single Test-RocmGfx211Leaf allowlist helper" + ) + assert re.search(r"@\('gfx120x-all',\s*'gfx1151',\s*'gfx1150'\)", text.lower()), ( + "Test-RocmGfx211Leaf should hold the gfx-2.11 allowlist" + ) + assert "$_pinGfx211 = Test-RocmGfx211Leaf" in text, ( + "setup.ps1 install-spec path should reuse Test-RocmGfx211Leaf, not " + "re-hardcode the allowlist (they must not diverge)" + ) + + def test_stack_py_allowlist(self): + text = STACK_PY.read_text(encoding = "utf-8").lower() + assert '"gfx120x-all", "gfx1151", "gfx1150"' in text, ( + "install_python_stack.py _ROCM_GFX_TORCH211_LEAVES not found / changed" + ) + + +class TestCudaLeafDigitParity: + """A wheel-family leaf is CUDA only when it is "cu" + digits (cu118/cu128/...). + A bare cu* glob wrongly catches mirror leaves like /custom or /current; when + that happened the venv was marked stale and rebuilt on every run. Every + installer must require a digit after "cu" in its family/CUDA classification.""" + + def test_stack_py_requires_cu_digit(self): + text = STACK_PY.read_text(encoding = "utf-8") + assert re.search(r'r"\^cu\[0-9\]"', text), ( + "install_python_stack.py _is_cuda_family_leaf must match ^cu[0-9]" + ) + + def test_setup_ps1_requires_cu_digit(self): + text = SETUP_PS1.read_text(encoding = "utf-8") + assert re.search(r"'\^cu\[0-9\]'", text), ( + "setup.ps1 Test-CudaFamilyLeaf must match ^cu[0-9], not a bare cu* glob" + ) + # The stale-venv branch must go through the digit-guarded helper. + assert "Test-CudaFamilyLeaf $_pinLeaf" in text, ( + "setup.ps1 stale check should classify CUDA via Test-CudaFamilyLeaf" + ) + + def test_install_ps1_requires_cu_digit_in_gpu_branch(self): + text = INSTALL_PS1.read_text(encoding = "utf-8") + assert re.search(r"'\^cu\[0-9\]'", text), ( + "install.ps1 Get-TauriGpuBranch must require a digit after cu" + ) + + def test_install_sh_requires_cu_digit_in_gpu_branch(self): + text = INSTALL_SH.read_text(encoding = "utf-8") + # The _tauri_gpu_branch cuda case must be cu[0-9]*, not a bare cu*. + assert re.search(r"cu\[0-9\]\*\)\s*echo \"cuda\"", text), ( + "install.sh _tauri_gpu_branch cuda case must be cu[0-9]*, not cu*" + ) diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py index 78fab998cd..bb71c1b4bb 100644 --- a/tests/studio/install/test_rocm_support.py +++ b/tests/studio/install/test_rocm_support.py @@ -569,19 +569,27 @@ class TestEnsureRocmTorch: _ensure_rocm_torch() mock_pip.assert_not_called() + @patch.object(stack_mod, "IS_WINDOWS", False) + @patch.object(stack_mod, "pip_install_try", return_value = True) @patch.object(stack_mod, "pip_install") @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False) @patch.object(stack_mod, "_has_rocm_gpu", return_value = True) @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 1)) - def test_torch_already_has_cuda_skips(self, mock_ver, mock_gpu, mock_nvidia, mock_pip): - """If torch already has CUDA, should skip ROCm reinstall.""" + def test_cuda_torch_on_amd_host_reinstalls( + self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try + ): + """A CUDA-only torch build is unusable on an AMD-only host, so it must be + reinstalled to ROCm (has_hip_torch is driven by the empty HIP marker, not + by treating the CUDA version string as a HIP marker).""" mock_probe = MagicMock() mock_probe.returncode = 0 - mock_probe.stdout = b"12.6\n" # CUDA version + # New single-line probe: empty HIP marker before "|" for a CUDA build. + mock_probe.stdout = b"|2.10.0+cu126\n" with patch("os.path.isdir", return_value = True): with patch("subprocess.run", return_value = mock_probe): _ensure_rocm_torch() - mock_pip.assert_not_called() + assert mock_pip.call_count == 1 + assert "rocm7.1" in str(mock_pip.call_args_list[0]) @patch.object(stack_mod, "pip_install") @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False) @@ -591,12 +599,34 @@ class TestEnsureRocmTorch: """If torch already has HIP, should skip ROCm reinstall.""" mock_probe = MagicMock() mock_probe.returncode = 0 - mock_probe.stdout = b"7.1.12345\n" # HIP version + mock_probe.stdout = b"7.1.12345|2.10.0+rocm7.1\n" # HIP marker + version with patch("os.path.isdir", return_value = True): with patch("subprocess.run", return_value = mock_probe): _ensure_rocm_torch() mock_pip.assert_not_called() + @patch.object(stack_mod, "pip_install") + @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False) + @patch.object(stack_mod, "_has_rocm_gpu", return_value = True) + @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 1)) + def test_cpu_torch_probe_line_not_read_as_hip( + self, mock_ver, mock_gpu, mock_nvidia, mock_pip + ): + """Regression: a CPU torch probe emits an EMPTY HIP marker before "|"; the + positional parse must keep that empty field so has_hip_torch stays False + (an earlier parse dropped the empty line and shifted the version into the + marker slot, wrongly reporting HIP and skipping the ROCm reinstall).""" + mock_probe = MagicMock() + mock_probe.returncode = 0 + mock_probe.stdout = b"|2.10.0+cpu\n" + # has_hip_torch False -> the AMD host reinstalls ROCm; assert we do NOT skip. + with patch("os.path.isdir", return_value = True): + with patch.object(stack_mod, "pip_install_try", return_value = True): + with patch("subprocess.run", return_value = mock_probe): + _ensure_rocm_torch() + assert mock_pip.call_count == 1 + assert "rocm7.1" in str(mock_pip.call_args_list[0]) + @patch.object(stack_mod, "IS_WINDOWS", False) @patch.object(stack_mod, "pip_install_try", return_value = True) @patch.object(stack_mod, "pip_install") @@ -724,7 +754,7 @@ class TestEnsureRocmTorch: mock_probe = MagicMock() mock_probe.returncode = 0 # HIP marker present (has_hip_torch=True) + installed +rocm6.4 wheel. - mock_probe.stdout = b"6.4.12345\n2.10.0+rocm6.4\n" + mock_probe.stdout = b"6.4.12345|2.10.0+rocm6.4\n" env = {"UNSLOTH_TORCH_INDEX_FAMILY": "rocm7.2"} with patch.dict(stack_mod.os.environ, env, clear = False): stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_URL", None) @@ -747,7 +777,7 @@ class TestEnsureRocmTorch: """A gfx* pin (2.11 line) over an installed pre-2.11 +rocm6.4 build reinstalls.""" mock_probe = MagicMock() mock_probe.returncode = 0 - mock_probe.stdout = b"6.4.12345\n2.10.0+rocm6.4\n" + mock_probe.stdout = b"6.4.12345|2.10.0+rocm6.4\n" env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx1151"} with patch.dict(stack_mod.os.environ, env, clear = False): stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None) @@ -774,7 +804,7 @@ class TestEnsureRocmTorch: (no false reinstall of a correct ROCm venv).""" mock_probe = MagicMock() mock_probe.returncode = 0 - mock_probe.stdout = b"7.2.12345\n2.11.0+rocm7.2\n" + mock_probe.stdout = b"7.2.12345|2.11.0+rocm7.2\n" env = {"UNSLOTH_TORCH_INDEX_FAMILY": "rocm7.2"} with patch.dict(stack_mod.os.environ, env, clear = False): stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_URL", None) @@ -794,20 +824,118 @@ class TestEnsureRocmTorch: any(str(a).startswith("torch") for a in _c.args) for _c in mock_pip.call_args_list ) + @patch.object(stack_mod, "IS_WINDOWS", False) + @patch.object(stack_mod, "pip_install_try", return_value = True) + @patch.object(stack_mod, "pip_install") + @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False) + @patch.object(stack_mod, "_has_rocm_gpu", return_value = True) + @patch.object(stack_mod, "_detect_rocm_version", return_value = (6, 4)) + def test_non211_gfx_pin_over_210_rocm_no_reinstall( + self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try + ): + """A gfx110X-all pin (NOT in the 2.11 allowlist) over a correct 2.10+rocm + wheel must NOT be flagged stale -- the install path uses the default <2.11 + specs for that arch, so re-flagging would reinstall-loop on every update.""" + mock_probe = MagicMock() + mock_probe.returncode = 0 + mock_probe.stdout = b"6.4.12345|2.10.0+rocm6.4\n" + env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx110X-all"} + with patch.dict(stack_mod.os.environ, env, clear = False): + stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None) + with patch("os.path.isdir", return_value = True): + with patch("subprocess.run", return_value = mock_probe): + _ensure_rocm_torch() + # has_hip_torch True + no mismatch -> torch must NOT be reinstalled. + assert not any( + any(str(a).startswith("torch") for a in _c.args) for _c in mock_pip.call_args_list + ) + + @patch.object(stack_mod, "IS_WINDOWS", False) + @patch.object(stack_mod, "pip_install_try", return_value = True) + @patch.object(stack_mod, "pip_install") + @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False) + @patch.object(stack_mod, "_has_rocm_gpu", return_value = True) + @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 2)) + def test_gfx_pin_over_generic_rocm211_reinstalls( + self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try + ): + """A gfx1151 pin over a GENERIC (two-part +rocm7.2) 2.11 wheel must reinstall + the AMD per-arch wheel -- even though both are torch 2.11, the generic wheel + is not the per-arch build the user pinned (Strix stays off the generic wheel).""" + mock_probe = MagicMock() + mock_probe.returncode = 0 + mock_probe.stdout = b"7.2.12345|2.11.0+rocm7.2\n" + env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx1151"} + with patch.dict(stack_mod.os.environ, env, clear = False): + stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None) + with patch("os.path.isdir", return_value = True): + with patch("subprocess.run", return_value = mock_probe): + with patch.object( + stack_mod, "_detect_amd_gfx_codes", side_effect = AssertionError + ): + _ensure_rocm_torch() + torch_call = str(mock_pip.call_args_list[0]) + assert "gfx1151" in torch_call + assert "torch>=2.11.0,<2.12.0" in torch_call + + @patch.object(stack_mod, "IS_WINDOWS", False) + @patch.object(stack_mod, "pip_install_try", return_value = True) + @patch.object(stack_mod, "pip_install") + @patch.object(stack_mod, "_has_usable_nvidia_gpu", return_value = False) + @patch.object(stack_mod, "_has_rocm_gpu", return_value = True) + @patch.object(stack_mod, "_detect_rocm_version", return_value = (7, 2)) + def test_gfx_pin_over_installed_perarch_no_reinstall( + self, mock_ver, mock_gpu, mock_nvidia, mock_pip, mock_pip_try + ): + """A gfx1151 pin over an already-installed AMD per-arch (+rocm7.13.0) wheel + must NOT reinstall torch -- once the correct per-arch wheel is present the + pin is satisfied, so `studio update` does not reinstall-loop.""" + mock_probe = MagicMock() + mock_probe.returncode = 0 + mock_probe.stdout = b"7.13.0|2.11.0+rocm7.13.0\n" + env = {"UNSLOTH_TORCH_INDEX_URL": "https://repo.amd.com/rocm/whl/gfx1151"} + with patch.dict(stack_mod.os.environ, env, clear = False): + stack_mod.os.environ.pop("UNSLOTH_TORCH_INDEX_FAMILY", None) + with patch("os.path.isdir", return_value = True): + with patch("subprocess.run", return_value = mock_probe): + with patch.object( + stack_mod, "_detect_amd_gfx_codes", side_effect = AssertionError + ): + _ensure_rocm_torch() + assert not any( + any(str(a).startswith("torch") for a in _c.args) for _c in mock_pip.call_args_list + ) + def test_rocm_pin_family_mismatch_helper(self): """_rocm_pin_family_mismatch: exact rocm compare, else the 2.11 line.""" f = stack_mod._rocm_pin_family_mismatch base = "https://download.pytorch.org/whl" + amd = "https://repo.amd.com/rocm/whl" # Exact rocm version comparison. assert f(f"{base}/rocm7.2", "2.11.0+rocm7.2") is False assert f(f"{base}/rocm7.2", "2.10.0+rocm6.4") is True assert f(f"{base}/rocm6.4", "2.10.0+rocm6.4") is False # gfx pin (2.11 line) vs installed release line. - assert f("https://repo.amd.com/rocm/whl/gfx1151", "2.10.0+rocm6.4") is True - assert f("https://repo.amd.com/rocm/whl/gfx1151", "2.11.0+rocm7.13.0") is False + assert f(f"{amd}/gfx1151", "2.10.0+rocm6.4") is True + assert f(f"{amd}/gfx1151", "2.11.0+rocm7.13.0") is False # rocm7.2 pin vs unreadable installed rocm version -> compare on 2.11 line. assert f(f"{base}/rocm7.2", "2.10.0") is True assert f(f"{base}/rocm7.2", "2.11.0") is False + # A 2.11-allowlist gfx pin over a GENERIC (two-part +rocm7.2) 2.11 wheel is a + # mismatch -- the user wants AMD's per-arch (three-part) wheel, not generic. + assert f(f"{amd}/gfx1151", "2.11.0+rocm7.2") is True + assert f(f"{amd}/gfx120X-all", "2.11.0+rocm7.2") is True + # ...but an already-installed per-arch (three-part) wheel is NOT re-flagged + # (no reinstall loop once the correct gfx wheel is present). + assert f(f"{amd}/gfx120X-all", "2.11.0+rocm7.13.0") is False + assert f(f"{amd}/gfx1150", "2.11.0+rocm7.13.0") is False + # A NON-2.11 gfx pin (gfx110X-all/gfx90a/gfx908) tracks the default <2.11 + # spec: a correct 2.10+rocm wheel is NOT a mismatch (no reinstall loop); + # a 2.11 build is (the arch's index does not publish 2.11 wheels). + assert f(f"{amd}/gfx110X-all", "2.10.0+rocm6.4") is False + assert f(f"{amd}/gfx90a", "2.10.0+rocm6.3") is False + assert f(f"{amd}/gfx908", "2.10.0+rocm7.0") is False + assert f(f"{amd}/gfx110X-all", "2.11.0+rocm7.2") is True @patch.object(stack_mod, "IS_WINDOWS", False) @patch.object(stack_mod, "pip_install_try", return_value = True) diff --git a/tests/studio/test_setup_pin_stale.ps1 b/tests/studio/test_setup_pin_stale.ps1 new file mode 100644 index 0000000000..6497f487d8 --- /dev/null +++ b/tests/studio/test_setup_pin_stale.ps1 @@ -0,0 +1,79 @@ +#!/usr/bin/env pwsh +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 +# Unit test for studio/setup.ps1's pinned-torch-index stale-venv helpers +# (Test-RocmGfx211Leaf, Test-CudaFamilyLeaf, Get-RocmPinStaleTags). Pure helpers, +# AST-extracted and run in-process -- no GPU/venv needed. Mirrors the Python +# _rocm_pin_family_mismatch / _is_cuda_family_leaf tests so both stay in lockstep. +# Run: pwsh -NoProfile -File tests/studio/test_setup_pin_stale.ps1 + +$ErrorActionPreference = "Stop" +$setupPath = [System.IO.Path]::Combine($PSScriptRoot, "..", "..", "studio", "setup.ps1") +$setupPath = (Resolve-Path $setupPath).Path + +# --- Parse setup.ps1 (also serves as a syntax gate) and extract the helpers --- +$tokens = $null; $errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile($setupPath, [ref]$tokens, [ref]$errors) +if ($errors) { $errors | ForEach-Object { $_.ToString() }; throw "setup.ps1 has parse errors" } + +foreach ($name in @("Test-RocmGfx211Leaf", "Test-CudaFamilyLeaf", "Get-RocmPinStaleTags")) { + $fn = $ast.FindAll({ param($n) + $n -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $n.Name -eq $name + }, $true) + if ($fn.Count -ne 1) { throw "expected exactly one $name in setup.ps1, found $($fn.Count)" } + # Pure helpers (no exit / external calls) -- safe to define in this scope. + Invoke-Expression $fn[0].Extent.Text +} + +$failures = 0 +function Check($name, $cond) { + if ($cond) { Write-Host " PASS $name" } + else { Write-Host " FAIL $name" -ForegroundColor Red; $script:failures++ } +} + +# A pinned gfx/rocm index is stale when Expected != Installed. +function IsStale($leaf, $ver) { + $t = Get-RocmPinStaleTags -PinLeaf $leaf -TorchVersion $ver + return $t.Expected -ne $t.Installed +} + +Write-Host "Test-RocmGfx211Leaf (the 2.11 gfx allowlist)" +Check "gfx1151 -> true" (Test-RocmGfx211Leaf "gfx1151") +Check "gfx1150 -> true" (Test-RocmGfx211Leaf "gfx1150") +Check "gfx120x-all -> true" (Test-RocmGfx211Leaf "gfx120x-all") +Check "gfx110x-all -> false" (-not (Test-RocmGfx211Leaf "gfx110x-all")) +Check "gfx90a -> false" (-not (Test-RocmGfx211Leaf "gfx90a")) +Check "gfx908 -> false" (-not (Test-RocmGfx211Leaf "gfx908")) + +Write-Host "Test-CudaFamilyLeaf (^cu[0-9])" +Check "cu118 -> true" (Test-CudaFamilyLeaf "cu118") +Check "cu128 -> true" (Test-CudaFamilyLeaf "cu128") +Check "cu130 -> true" (Test-CudaFamilyLeaf "cu130") +Check "custom -> false" (-not (Test-CudaFamilyLeaf "custom")) +Check "current -> false" (-not (Test-CudaFamilyLeaf "current")) +Check "cpu -> false" (-not (Test-CudaFamilyLeaf "cpu")) +Check "empty -> false" (-not (Test-CudaFamilyLeaf "")) + +Write-Host "Get-RocmPinStaleTags (mirror of _rocm_pin_family_mismatch)" +# Exact rocm version comparison. +Check "rocm7.2 pin + 2.11.0+rocm7.2 -> not stale" (-not (IsStale "rocm7.2" "2.11.0+rocm7.2")) +Check "rocm7.2 pin + 2.10.0+rocm6.4 -> stale" (IsStale "rocm7.2" "2.10.0+rocm6.4") +Check "rocm6.4 pin + 2.10.0+rocm6.4 -> not stale" (-not (IsStale "rocm6.4" "2.10.0+rocm6.4")) +# rocm pin vs unreadable installed rocm version -> compare on the 2.11 line. +Check "rocm7.2 pin + 2.10.0 -> stale" (IsStale "rocm7.2" "2.10.0") +Check "rocm7.2 pin + 2.11.0 -> not stale" (-not (IsStale "rocm7.2" "2.11.0")) +# 2.11-allowlist gfx pin: per-arch (three-part) wheel is satisfied, generic is stale. +Check "gfx1151 pin + 2.11.0+rocm7.13.0 -> not stale" (-not (IsStale "gfx1151" "2.11.0+rocm7.13.0")) +Check "gfx1150 pin + 2.11.0+rocm7.13.0 -> not stale" (-not (IsStale "gfx1150" "2.11.0+rocm7.13.0")) +Check "gfx120x-all pin + 2.11.0+rocm7.13.0 -> not stale" (-not (IsStale "gfx120x-all" "2.11.0+rocm7.13.0")) +Check "gfx1151 pin + 2.11.0+rocm7.2 (generic) -> stale" (IsStale "gfx1151" "2.11.0+rocm7.2") +Check "gfx1151 pin + 2.10.0+rocm6.4 -> stale" (IsStale "gfx1151" "2.10.0+rocm6.4") +# Non-2.11 gfx pin (gfx110X-all/gfx90a/gfx908): a valid <2.11 wheel is NOT stale. +Check "gfx110x-all pin + 2.10.0+rocm6.4 -> not stale" (-not (IsStale "gfx110x-all" "2.10.0+rocm6.4")) +Check "gfx90a pin + 2.10.0+rocm6.3 -> not stale" (-not (IsStale "gfx90a" "2.10.0+rocm6.3")) +Check "gfx908 pin + 2.10.0+rocm7.0 -> not stale" (-not (IsStale "gfx908" "2.10.0+rocm7.0")) +Check "gfx110x-all pin + 2.11.0+rocm7.2 -> stale" (IsStale "gfx110x-all" "2.11.0+rocm7.2") + +Write-Host "" +if ($failures -gt 0) { Write-Host "$failures check(s) FAILED" -ForegroundColor Red; exit 1 } +Write-Host "All checks passed" -ForegroundColor Green