Studio: respect explicit --numa and reorder the NUMA total-RAM check
Follow-up review on the CPU-only / NUMA hardening: - Respect an explicit user --numa policy. Auto-interleave wraps the process with a numactl argv prefix the user cannot override, so when the user sets --numa we now skip the wrapper (and --numa distribute) instead of forcing interleaving over their placement. - Check the total-RAM condition before numactl availability in decide_interleave, so a model too large for total free RAM gets the smaller-quant / free-memory guidance instead of being told to install numactl, which could never help. Unit tests cover both cases.
This commit is contained in:
parent
d526db9065
commit
2ef8fb6696
4 changed files with 34 additions and 10 deletions
|
|
@ -6048,10 +6048,14 @@ class LlamaCppBackend:
|
|||
except Exception:
|
||||
_numa_footprint = _resident
|
||||
_numa = decide_interleave(_numa_footprint, cpu_only = _cpu_only)
|
||||
if _numa.interleave:
|
||||
if _numa.interleave and _extra_args_set_any_flag(extra_args, {"--numa"}):
|
||||
# User set an explicit --numa policy: respect it. The numactl wrap
|
||||
# is an argv prefix they can't override, so skip it (and --numa
|
||||
# distribute) rather than force interleaving over their choice.
|
||||
logger.info("NUMA: user --numa set; leaving auto-interleave off")
|
||||
elif _numa.interleave:
|
||||
self._numa_prefix = list(_numa.prefix)
|
||||
if not _extra_args_set_any_flag(extra_args, {"--numa"}):
|
||||
cmd.extend(["--numa", "distribute"])
|
||||
cmd.extend(["--numa", "distribute"])
|
||||
logger.info("NUMA: %s", _numa.reason)
|
||||
elif _cpu_only and (
|
||||
"numactl` is not installed" in _numa.reason
|
||||
|
|
|
|||
|
|
@ -135,6 +135,16 @@ def decide_interleave(
|
|||
f"(~{largest} MiB); keeping local placement",
|
||||
)
|
||||
|
||||
# Impossible across all nodes regardless of numactl: surface the smaller-quant /
|
||||
# free-memory path before the numactl hint, so a too-big model is not told to
|
||||
# install numactl when interleaving could never make it fit.
|
||||
if model_mib > total:
|
||||
return InterleaveDecision(
|
||||
False,
|
||||
f"model ~{model_mib} MiB exceeds total free RAM across all nodes "
|
||||
f"(~{total} MiB); interleave cannot help -- free memory or use a smaller quant",
|
||||
)
|
||||
|
||||
avail = numactl_available() if has_numactl is None else has_numactl
|
||||
if not avail:
|
||||
# Needed but unavailable: surface it; caller decides whether to block.
|
||||
|
|
@ -146,13 +156,6 @@ def decide_interleave(
|
|||
f"numactl`) or the model may fail to fit a single node.",
|
||||
)
|
||||
|
||||
if model_mib > total:
|
||||
return InterleaveDecision(
|
||||
False,
|
||||
f"model ~{model_mib} MiB exceeds total free RAM across all nodes "
|
||||
f"(~{total} MiB); interleave cannot help -- free memory or use a smaller quant",
|
||||
)
|
||||
|
||||
return InterleaveDecision(
|
||||
True,
|
||||
f"model ~{model_mib} MiB exceeds the largest NUMA node's free RAM "
|
||||
|
|
|
|||
|
|
@ -182,6 +182,14 @@ def test_numa_surfaces_total_ram_failure():
|
|||
assert '"interleave cannot help" in _numa.reason' in src
|
||||
|
||||
|
||||
def test_explicit_user_numa_skips_auto_interleave_prefix():
|
||||
"""An explicit user --numa must skip the numactl argv prefix (which user extra args
|
||||
can't override), not just the --numa distribute flag (PR review fix)."""
|
||||
src = _load_model_src()
|
||||
assert 'if _numa.interleave and _extra_args_set_any_flag(extra_args, {"--numa"}):' in src
|
||||
assert "leaving auto-interleave off" in src
|
||||
|
||||
|
||||
def test_extra_args_forces_cpu_offload_helper():
|
||||
"""The zero-offload detector: -ngl 0 / --n-gpu-layers 0 / --gpu-layers 0 (last wins)."""
|
||||
from core.inference.llama_cpp import _extra_args_forces_cpu_offload as f
|
||||
|
|
|
|||
|
|
@ -87,6 +87,15 @@ def test_numactl_missing_surfaces_actionable_warning():
|
|||
assert "numactl` is not installed" in d.reason
|
||||
|
||||
|
||||
def test_too_big_and_numactl_missing_prefers_total_ram_message():
|
||||
# Impossible across all nodes AND no numactl: the total-RAM guidance must win, so the
|
||||
# user is not told to install numactl when interleaving could never help (PR review fix).
|
||||
d = decide_interleave(800 * _GiB, cpu_only = True, topology = _USER_TOPO, has_numactl = False)
|
||||
assert d.interleave is False
|
||||
assert "exceeds total free RAM" in d.reason
|
||||
assert "numactl` is not installed" not in d.reason
|
||||
|
||||
|
||||
def test_unknown_model_size_does_not_force_interleave():
|
||||
for size in (None, 0, -1):
|
||||
d = decide_interleave(size, cpu_only = True, topology = _USER_TOPO, has_numactl = True)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue