From 2ef8fb6696b56dd4887bd5c3e4c2704fb32dbfbd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 29 Jun 2026 07:39:36 +0000 Subject: [PATCH] Studio: respect explicit --numa and reorder the NUMA total-RAM check Follow-up review on the CPU-only / NUMA hardening: - Respect an explicit user --numa policy. Auto-interleave wraps the process with a numactl argv prefix the user cannot override, so when the user sets --numa we now skip the wrapper (and --numa distribute) instead of forcing interleaving over their placement. - Check the total-RAM condition before numactl availability in decide_interleave, so a model too large for total free RAM gets the smaller-quant / free-memory guidance instead of being told to install numactl, which could never help. Unit tests cover both cases. --- studio/backend/core/inference/llama_cpp.py | 10 +++++++--- studio/backend/core/inference/numa.py | 17 ++++++++++------- studio/backend/tests/test_cpu_only_defaults.py | 8 ++++++++ studio/backend/tests/test_numa_interleave.py | 9 +++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index e21e4dde16..f21124a392 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -6048,10 +6048,14 @@ class LlamaCppBackend: except Exception: _numa_footprint = _resident _numa = decide_interleave(_numa_footprint, cpu_only = _cpu_only) - if _numa.interleave: + if _numa.interleave and _extra_args_set_any_flag(extra_args, {"--numa"}): + # User set an explicit --numa policy: respect it. The numactl wrap + # is an argv prefix they can't override, so skip it (and --numa + # distribute) rather than force interleaving over their choice. + logger.info("NUMA: user --numa set; leaving auto-interleave off") + elif _numa.interleave: self._numa_prefix = list(_numa.prefix) - if not _extra_args_set_any_flag(extra_args, {"--numa"}): - cmd.extend(["--numa", "distribute"]) + cmd.extend(["--numa", "distribute"]) logger.info("NUMA: %s", _numa.reason) elif _cpu_only and ( "numactl` is not installed" in _numa.reason diff --git a/studio/backend/core/inference/numa.py b/studio/backend/core/inference/numa.py index ba41331726..d53fafc5a4 100644 --- a/studio/backend/core/inference/numa.py +++ b/studio/backend/core/inference/numa.py @@ -135,6 +135,16 @@ def decide_interleave( f"(~{largest} MiB); keeping local placement", ) + # Impossible across all nodes regardless of numactl: surface the smaller-quant / + # free-memory path before the numactl hint, so a too-big model is not told to + # install numactl when interleaving could never make it fit. + if model_mib > total: + return InterleaveDecision( + False, + f"model ~{model_mib} MiB exceeds total free RAM across all nodes " + f"(~{total} MiB); interleave cannot help -- free memory or use a smaller quant", + ) + avail = numactl_available() if has_numactl is None else has_numactl if not avail: # Needed but unavailable: surface it; caller decides whether to block. @@ -146,13 +156,6 @@ def decide_interleave( f"numactl`) or the model may fail to fit a single node.", ) - if model_mib > total: - return InterleaveDecision( - False, - f"model ~{model_mib} MiB exceeds total free RAM across all nodes " - f"(~{total} MiB); interleave cannot help -- free memory or use a smaller quant", - ) - return InterleaveDecision( True, f"model ~{model_mib} MiB exceeds the largest NUMA node's free RAM " diff --git a/studio/backend/tests/test_cpu_only_defaults.py b/studio/backend/tests/test_cpu_only_defaults.py index b5a3ea643e..e03061e65b 100644 --- a/studio/backend/tests/test_cpu_only_defaults.py +++ b/studio/backend/tests/test_cpu_only_defaults.py @@ -182,6 +182,14 @@ def test_numa_surfaces_total_ram_failure(): assert '"interleave cannot help" in _numa.reason' in src +def test_explicit_user_numa_skips_auto_interleave_prefix(): + """An explicit user --numa must skip the numactl argv prefix (which user extra args + can't override), not just the --numa distribute flag (PR review fix).""" + src = _load_model_src() + assert 'if _numa.interleave and _extra_args_set_any_flag(extra_args, {"--numa"}):' in src + assert "leaving auto-interleave off" in src + + def test_extra_args_forces_cpu_offload_helper(): """The zero-offload detector: -ngl 0 / --n-gpu-layers 0 / --gpu-layers 0 (last wins).""" from core.inference.llama_cpp import _extra_args_forces_cpu_offload as f diff --git a/studio/backend/tests/test_numa_interleave.py b/studio/backend/tests/test_numa_interleave.py index f496530c6f..92377ce9db 100644 --- a/studio/backend/tests/test_numa_interleave.py +++ b/studio/backend/tests/test_numa_interleave.py @@ -87,6 +87,15 @@ def test_numactl_missing_surfaces_actionable_warning(): assert "numactl` is not installed" in d.reason +def test_too_big_and_numactl_missing_prefers_total_ram_message(): + # Impossible across all nodes AND no numactl: the total-RAM guidance must win, so the + # user is not told to install numactl when interleaving could never help (PR review fix). + d = decide_interleave(800 * _GiB, cpu_only = True, topology = _USER_TOPO, has_numactl = False) + assert d.interleave is False + assert "exceeds total free RAM" in d.reason + assert "numactl` is not installed" not in d.reason + + def test_unknown_model_size_does_not_force_interleave(): for size in (None, 0, -1): d = decide_interleave(size, cpu_only = True, topology = _USER_TOPO, has_numactl = True)