Studio: respect explicit --numa and reorder the NUMA total-RAM check

Follow-up review on the CPU-only / NUMA hardening:

- Respect an explicit user --numa policy. Auto-interleave wraps the process with
  a numactl argv prefix the user cannot override, so when the user sets --numa we
  now skip the wrapper (and --numa distribute) instead of forcing interleaving
  over their placement.
- Check the total-RAM condition before numactl availability in decide_interleave,
  so a model too large for total free RAM gets the smaller-quant / free-memory
  guidance instead of being told to install numactl, which could never help.

Unit tests cover both cases.
This commit is contained in:
Daniel Han 2026-06-29 07:39:36 +00:00
commit 2ef8fb6696
4 changed files with 34 additions and 10 deletions

View file

@ -6048,10 +6048,14 @@ class LlamaCppBackend:
except Exception:
_numa_footprint = _resident
_numa = decide_interleave(_numa_footprint, cpu_only = _cpu_only)
if _numa.interleave:
if _numa.interleave and _extra_args_set_any_flag(extra_args, {"--numa"}):
# User set an explicit --numa policy: respect it. The numactl wrap
# is an argv prefix they can't override, so skip it (and --numa
# distribute) rather than force interleaving over their choice.
logger.info("NUMA: user --numa set; leaving auto-interleave off")
elif _numa.interleave:
self._numa_prefix = list(_numa.prefix)
if not _extra_args_set_any_flag(extra_args, {"--numa"}):
cmd.extend(["--numa", "distribute"])
cmd.extend(["--numa", "distribute"])
logger.info("NUMA: %s", _numa.reason)
elif _cpu_only and (
"numactl` is not installed" in _numa.reason

View file

@ -135,6 +135,16 @@ def decide_interleave(
f"(~{largest} MiB); keeping local placement",
)
# Impossible across all nodes regardless of numactl: surface the smaller-quant /
# free-memory path before the numactl hint, so a too-big model is not told to
# install numactl when interleaving could never make it fit.
if model_mib > total:
return InterleaveDecision(
False,
f"model ~{model_mib} MiB exceeds total free RAM across all nodes "
f"(~{total} MiB); interleave cannot help -- free memory or use a smaller quant",
)
avail = numactl_available() if has_numactl is None else has_numactl
if not avail:
# Needed but unavailable: surface it; caller decides whether to block.
@ -146,13 +156,6 @@ def decide_interleave(
f"numactl`) or the model may fail to fit a single node.",
)
if model_mib > total:
return InterleaveDecision(
False,
f"model ~{model_mib} MiB exceeds total free RAM across all nodes "
f"(~{total} MiB); interleave cannot help -- free memory or use a smaller quant",
)
return InterleaveDecision(
True,
f"model ~{model_mib} MiB exceeds the largest NUMA node's free RAM "

View file

@ -182,6 +182,14 @@ def test_numa_surfaces_total_ram_failure():
assert '"interleave cannot help" in _numa.reason' in src
def test_explicit_user_numa_skips_auto_interleave_prefix():
"""An explicit user --numa must skip the numactl argv prefix (which user extra args
can't override), not just the --numa distribute flag (PR review fix)."""
src = _load_model_src()
assert 'if _numa.interleave and _extra_args_set_any_flag(extra_args, {"--numa"}):' in src
assert "leaving auto-interleave off" in src
def test_extra_args_forces_cpu_offload_helper():
"""The zero-offload detector: -ngl 0 / --n-gpu-layers 0 / --gpu-layers 0 (last wins)."""
from core.inference.llama_cpp import _extra_args_forces_cpu_offload as f

View file

@ -87,6 +87,15 @@ def test_numactl_missing_surfaces_actionable_warning():
assert "numactl` is not installed" in d.reason
def test_too_big_and_numactl_missing_prefers_total_ram_message():
# Impossible across all nodes AND no numactl: the total-RAM guidance must win, so the
# user is not told to install numactl when interleaving could never help (PR review fix).
d = decide_interleave(800 * _GiB, cpu_only = True, topology = _USER_TOPO, has_numactl = False)
assert d.interleave is False
assert "exceeds total free RAM" in d.reason
assert "numactl` is not installed" not in d.reason
def test_unknown_model_size_does_not_force_interleave():
for size in (None, 0, -1):
d = decide_interleave(size, cpu_only = True, topology = _USER_TOPO, has_numactl = True)