ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: stop the llama.cpp graph-scheduler abort reload loop and harden CPU-only / multi-NUMA GGUF inference

Browse source 
On a CPU-only host, large MLA / sparse-attention / MTP GGUFs (e.g. GLM-5.2) abort in
llama.cpp's graph scheduler (GGML_ASSERT(*cur_backend_id != -1) in
ggml_backend_sched_split_graph) because the backend cannot run an op in the graph.
Studio showed a generic "invalid GGUF or out of memory" message and, since a startup
crash 500s the load, the UI replayed /load and re-read the whole model (hundreds of GB)
into the same crash on a loop while sitting at 99%.

Changes (all in studio/backend):
- Detect the scheduler abort, memo (binary, model) as non-retryable, and fail the next
  /load fast with an actionable message (mirrors the existing tensor-split memo).
- CPU-only safe defaults (user flags still win): --fit off (its estimator hits the same
  abort, llama.cpp #21932, and mis-counts MTP KV #23472/#24117), --flash-attn off, and
  an auto context capped to a RAM-aware ceiling.
- NUMA auto-interleave: wrap with numactl --interleave=all when the model overflows the
  largest node's free RAM but fits across all nodes.
- CPU RAM preflight warning when weights exceed available RAM.

GPU and Apple paths are unchanged. Adds unit tests for the classifier, the memo, the
NUMA decision, and the CPU-only command defaults.
This commit is contained in:
Daniel Han 2026-06-29 05:18:48 +00:00
commit afbaba97bf
5 changed files with 767 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

173
studio/backend/core/inference/llama_cpp.py
View file

@ -811,6 +811,11 @@ _APPLE_UNIFIED_MEMORY_FRACTION = 0.85
# reserve (_estimate_mtp_overhead_bytes). Applied to both the fit budget and pin.
_MTP_VRAM_RESERVE_FRAC = 0.05
# CPU-only: cap an auto context to this ceiling (a full native context puts tens to
# hundreds of GB of KV + MTP reserve in RAM), then fit it to RAM. Explicit -c wins.
_CPU_CTX_AUTO_CEILING = 32768
_CPU_RAM_BUDGET_FRAC = 0.9 # RAM headroom for compute buffers + OS
def _kv_bytes_per_elem(cache_type: Optional[str]) -> float:
"""Bytes per KV-cache element for a llama.cpp cache type (f16 default)."""
@ -1234,6 +1239,8 @@ class LlamaCppBackend:
def __init__(self):
self._process: Optional[subprocess.Popen] = None
# Spawn-time argv prefix (e.g. numactl --interleave=all); recomputed per load.
self._numa_prefix: list[str] = []
self._port: Optional[int] = None
self._model_identifier: Optional[str] = None
self._gguf_path: Optional[str] = None
@ -2469,6 +2476,24 @@ class LlamaCppBackend:
if key is not None:
cls._tensor_split_abort_keys.add(key)
# (binary, mtime, model) that aborted in the ggml graph scheduler this process
# (GGML_ASSERT(*cur_backend_id != -1)): an unsupported op, so reloading just
# repeats the crash. Keyed like the tensor-split memo; mtime drops it on update.
_sched_reserve_abort_keys: set[tuple[str, int, str]] = set()
@classmethod
def _sched_reserve_aborts(cls, binary: Optional[str], model: Optional[str]) -> bool:
"""True if (binary, model) aborted in graph-scheduler reserve this session."""
key = cls._tensor_split_cache_key(binary, model)
return key is not None and key in cls._sched_reserve_abort_keys
@classmethod
def _record_sched_reserve_abort(cls, binary: Optional[str], model: Optional[str]) -> None:
"""Remember a (binary, model) that aborts in graph-scheduler reserve."""
key = cls._tensor_split_cache_key(binary, model)
if key is not None:
cls._sched_reserve_abort_keys.add(key)
@staticmethod
def _windows_pip_nvidia_dll_dirs(prefix: str) -> list[str]:
"""Return DLL dirs from pip-installed CUDA wheels under
@ -4119,6 +4144,10 @@ class LlamaCppBackend:
"settings and reload."
)
# ggml graph-scheduler abort: surface the real cause, not the generic fallback.
if LlamaCppBackend._is_sched_reserve_abort(output or ""):
return LlamaCppBackend._sched_reserve_abort_message()
# Detect Ollama source up front so the arch branch can keep the
# Ollama hint instead of the generic "unsupported arch" message.
gguf = gguf_path or ""
@ -4395,6 +4424,41 @@ class LlamaCppBackend:
# the split-axis enum token, unique to this assert (not the source file).
return "split_axis" in text
@staticmethod
def _is_sched_reserve_abort(output: str) -> bool:
"""True for the ggml graph-scheduler abort GGML_ASSERT(*cur_backend_id != -1):
no backend can run an op in the graph (e.g. an MLA/sparse-attention/MTP op
unimplemented on this build). Needs a ggml-abort marker plus a scheduler marker
(matched on the backtrace frames, which outlive the [New LWP] dump in a short
tail). Excludes the #6415 split-axis abort. stderr is merged into output."""
text = (output or "").lower()
if "ggml_assert" not in text and "ggml_abort" not in text:
return False
# #6415 split-axis abort shares the frame but is handled by the tensor latch.
if "split_axis" in text:
return False
return (
"cur_backend_id" in text
or "ggml_backend_sched_split_graph" in text
or "sched_reserve" in text
or "graph_reserve" in text
)
@staticmethod
def _sched_reserve_abort_message() -> str:
"""Actionable message for the graph-scheduler abort (classifier + fail-fast guard)."""
return (
"llama.cpp aborted while reserving the compute graph "
"(GGML_ASSERT(*cur_backend_id != -1) in ggml_backend_sched_split_graph): "
"the active backend cannot run an operation in this model's graph. This "
"usually means a newer attention variant (MLA / sparse-attention / MTP) "
"is not implemented in this llama.cpp build for the backend you're using "
"(commonly CPU-only). Disabling flash attention did not help. Try: run "
"`unsloth studio update` for a newer llama.cpp, use a different "
"quantization, run on a supported GPU/backend, or disable speculative "
"decoding / MTP and lower the context length."
)
@staticmethod
def _is_signal_crash(returncode: Optional[int]) -> bool:
"""True only on a hard fault (SIGSEGV/SIGABRT/SIGILL/SIGFPE/SIGBUS or a
@ -4516,12 +4580,13 @@ class LlamaCppBackend:
logger.debug(f"Could not open llama-server log file: {e}")
self._llama_log_path = None
# Log the argv per attempt (the text-only mmproj retry re-enters here
# with --mmproj stripped), redacting the API key.
logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}")
# Log the argv per attempt (mmproj retry re-enters with --mmproj stripped),
# redacting the API key. Prepend _numa_prefix so the log matches what runs.
_run_cmd = [*self._numa_prefix, *cmd]
logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(_run_cmd))}")
self._process = subprocess.Popen(
cmd,
_run_cmd,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
@ -4652,6 +4717,17 @@ class LlamaCppBackend:
# GGUF uses the diffusion runner, and its arch is only known after the header.
binary = self._find_llama_server_binary()
# Fail fast if this (binary, model) already aborted in the graph scheduler
# this session: reloading just re-reads the weights into the same crash
# (a startup crash 500s and the UI replays /load).
if LlamaCppBackend._sched_reserve_aborts(binary, model_identifier):
logger.warning(
"Skipping reload of '%s': it already aborted in the llama.cpp "
"graph scheduler this session (unsupported op on this backend).",
model_identifier,
)
raise RuntimeError(self._sched_reserve_abort_message())
# ── Phase 2: download (NO lock held, so cancel can proceed) ──
# mtp_draft_path arrives set for local Gemma loads (detected
# sibling); for -hf loads it's None here and resolved just below.
@ -4868,6 +4944,11 @@ class LlamaCppBackend:
# Layer-fallback min GPUs; raised below on a tensor downgrade. Bound
# before the try so the --fit-on except path still has it (no UnboundLocal).
_layer_min_gpus = 1
# CPU-only (no discrete GPU, not Apple Metal): drives safe defaults
# below since no GPU/Apple fit branch runs. Bound before the try (the
# --fit except path needs it); refined once gpus is known.
from utils.hardware import is_apple_silicon as _is_apple_silicon
_cpu_only = False
try:
gguf_size = self._get_gguf_size_bytes(model_path)
# Include GPU-loaded mmproj in the fit budget (#5825).
@ -4880,6 +4961,7 @@ class LlamaCppBackend:
_gpu_mem = self._get_gpu_memory()
gpus = [(idx, free) for idx, free, _t in _gpu_mem]
total_by_idx = {idx: total for idx, _f, total in _gpu_mem}
_cpu_only = (not gpus) and not _is_apple_silicon()
def _gpu_usable(g, frac = _CTX_FIT_VRAM_FRACTION):
# Per-GPU usable budget for ranking: free - (1-frac)*total.
@ -5581,6 +5663,53 @@ class LlamaCppBackend:
except Exception as e:
logger.debug(f"mmproj audio-capability read failed: {e}")
# CPU-only safe defaults (user extra_args win, appended last): no --fit
# (its graph-reserve estimator hits the same abort, llama.cpp #21932, and
# mis-counts MTP KV #23472/#24117; offloads nothing on CPU) and flash-attn
# off (CPU FA is unsafe for large MLA/sparse/MTP graphs).
_flash_default = "off" if _cpu_only else "on"
if _cpu_only and use_fit:
use_fit = False
logger.info("CPU-only host: launching with --fit off and --flash-attn off.")
if _cpu_only:
_avail_mib = self._available_system_memory_mib()
# Preflight: weights over total RAM get OS-killed mid-load (interleave
# spreads across nodes but can't beat the total).
if _avail_mib and model_size and model_size > _avail_mib * 1024 * 1024:
logger.warning(
"CPU-only memory preflight: model weights ~%.0f GB exceed "
"available RAM ~%.0f GB; loading may be OS-killed.",
model_size / (1024**3),
_avail_mib / 1024,
)
# Cap an auto context to a RAM-aware ceiling (explicit -c is honored).
if requested_ctx <= 0 and effective_ctx > _CPU_CTX_AUTO_CEILING:
_cpu_cap = _CPU_CTX_AUTO_CEILING
try:
if _avail_mib and model_size and self._can_estimate_kv():
_fit = self._fit_context_to_vram(
requested_ctx = _CPU_CTX_AUTO_CEILING,
available_mib = _avail_mib,
model_size_bytes = model_size,
cache_type_kv = cache_type_kv,
min_ctx = 4096,
n_parallel = n_parallel,
kv_on_gpu = True, # KV lives in the RAM budget we fit
mtp_engaged = True, # flat reserve; no GPU draft here
budget_frac = _CPU_RAM_BUDGET_FRAC,
)
_cpu_cap = max(4096, min(_CPU_CTX_AUTO_CEILING, _fit))
except Exception as _cap_exc: # best-effort; fall back to ceiling
logger.debug("CPU context-fit failed; using ceiling: %s", _cap_exc)
if _cpu_cap < effective_ctx:
logger.info(
"CPU-only: capping context %d -> %d (set -c to override).",
effective_ctx,
_cpu_cap,
)
effective_ctx = _cpu_cap
cmd = [
binary,
"-m",
@ -5592,7 +5721,7 @@ class LlamaCppBackend:
"--parallel",
str(n_parallel),
"--flash-attn",
"on", # Force flash attention for speed
_flash_default, # CPU-only: off; GPU: on for speed
# Error out at n_ctx instead of silently rotating the KV cache; frontend catches it and points the user at "Context Length".
"--no-context-shift",
]
@ -5828,7 +5957,26 @@ class LlamaCppBackend:
cmd.extend(str(a) for a in extra_args)
logger.info(f"Appending user extra args to llama-server: {list(extra_args)}")
logger.info(f"Starting llama-server: {' '.join(self._redacted_cmd_for_log(cmd))}")
# NUMA auto-interleave: wrap with `numactl --interleave=all` when the
# model overflows one node but fits across all (else first-touch
# thrashes/OOMs). Applied at the Popen sites; decided before any spawn.
self._numa_prefix = []
try:
from core.inference.numa import decide_interleave
_numa = decide_interleave(model_size, cpu_only=_cpu_only)
if _numa.interleave:
self._numa_prefix = list(_numa.prefix)
logger.info("NUMA: %s", _numa.reason)
elif _cpu_only and "numactl` is not installed" in _numa.reason:
logger.warning("NUMA: %s", _numa.reason)
except Exception as _numa_exc: # never block a load on the NUMA probe
logger.debug("NUMA interleave probe failed: %s", _numa_exc)
logger.info(
"Starting llama-server: "
f"{' '.join(self._redacted_cmd_for_log([*self._numa_prefix, *cmd]))}"
)
# Library paths so llama-server finds its shared libs and CUDA DLLs.
env = self._llama_server_env_for_binary(binary)
@ -5969,9 +6117,11 @@ class LlamaCppBackend:
# Best-effort; never block the load on logging.
logger.debug(f"Could not open llama-server log file: {e}")
self._llama_log_path = None
# _last_spawn_cmd stays un-prefixed (retry helpers slice it);
# the NUMA prefix goes on the actual argv only, so retries don't double it.
_last_spawn_cmd = list(run_cmd)
self._process = subprocess.Popen(
run_cmd,
[*self._numa_prefix, *run_cmd],
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
text = True,
@ -6205,6 +6355,15 @@ class LlamaCppBackend:
out = "\n".join(self._stdout_lines[-50:])
# Read the crash code before _kill_process() clears _process.
_crash_rc = self._process.poll() if self._process is not None else None
# Graph-scheduler abort (flash-off retry already failed): memo it so
# the next /load fails fast. Wider slice as the GGML_ASSERT line can
# scroll past the [New LWP] dump (backtrace markers stay in the tail).
if (
not self._cancel_event.is_set()
and (self._is_signal_crash(_crash_rc) or self._is_abort_exit(_crash_rc))
and self._is_sched_reserve_abort("\n".join(self._stdout_lines[-200:]))
):
LlamaCppBackend._record_sched_reserve_abort(binary, model_identifier)
self._kill_process()
# The #6415 split-axis abort is latched earlier (first spawn).
# Skip if a cancel/unload is pending (mirrors the MTP guard).

160
studio/backend/core/inference/numa.py Normal file
View file

@ -0,0 +1,160 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""NUMA topology detection and an auto-interleave decision for CPU-only launches.
Linux's first-touch policy faults a model's pages onto one node; if the GGUF is larger
than that node's free RAM the load thrashes/fails despite ample total RAM.
`numactl --interleave=all` spreads pages across nodes (llama.cpp discussion #19102).
Pure stdlib, Linux-only (no-op decision elsewhere), side-effect free.
"""
from __future__ import annotations
import shutil
from dataclasses import dataclass, field
from pathlib import Path
_NODE_ROOT = Path("/sys/devices/system/node")
@dataclass(frozen=True)
class NumaTopology:
"""Per-node free memory (MemFree, matching `numactl --hardware`'s 'node N free')."""
node_free_mib: dict[int, int] = field(default_factory=dict)
@property
def node_count(self) -> int:
return len(self.node_free_mib)
@property
def total_free_mib(self) -> int:
return sum(self.node_free_mib.values())
@property
def largest_node_free_mib(self) -> int:
return max(self.node_free_mib.values(), default=0)
def _parse_online(spec: str) -> list[int]:
"""Parse a sysfs cpulist-style range, e.g. '0-1' or '0,2-3', into node ids."""
out: list[int] = []
for part in spec.strip().split(","):
part = part.strip()
if not part:
continue
if "-" in part:
lo, _, hi = part.partition("-")
try:
out.extend(range(int(lo), int(hi) + 1))
except ValueError:
continue
else:
try:
out.append(int(part))
except ValueError:
continue
return out
def _node_memfree_mib(node: int) -> int | None:
"""MemFree for one node from /sys/.../nodeN/meminfo, in MiB (kB -> MiB)."""
try:
text = (_NODE_ROOT / f"node{node}" / "meminfo").read_text()
except OSError:
return None
for line in text.splitlines():
# "Node 0 MemFree: 465594 kB"
if "MemFree:" in line:
parts = line.split()
try:
kb = int(parts[parts.index("MemFree:") + 1])
except (ValueError, IndexError):
return None
return kb // 1024
return None
def read_numa_topology() -> NumaTopology:
"""Read NUMA node free memory from sysfs. Empty topology on non-Linux / no sysfs /
single-node (a single node is reported but callers treat node_count <= 1 as 'no
interleave needed')."""
try:
online = (_NODE_ROOT / "online").read_text()
except OSError:
return NumaTopology()
free: dict[int, int] = {}
for node in _parse_online(online):
mib = _node_memfree_mib(node)
if mib is not None:
free[node] = mib
return NumaTopology(node_free_mib=free)
def numactl_available() -> bool:
return shutil.which("numactl") is not None
@dataclass(frozen=True)
class InterleaveDecision:
interleave: bool
reason: str
prefix: tuple[str, ...] = () # argv prefix to apply, e.g. ("numactl", "--interleave=all")
def decide_interleave(
model_size_bytes: int | None,
*,
cpu_only: bool,
topology: NumaTopology | None = None,
has_numactl: bool | None = None,
) -> InterleaveDecision:
"""Interleave only when CPU-only, multi-node, and the model exceeds the largest
node's free RAM but fits across all nodes; otherwise leave placement local.
model_size_bytes (GGUF weights) is the conservative proxy for the footprint."""
if not cpu_only:
return InterleaveDecision(False, "not cpu-only; leaving NUMA placement to the OS")
if not model_size_bytes or model_size_bytes <= 0:
return InterleaveDecision(False, "model size unknown; not forcing interleave")
topo = topology if topology is not None else read_numa_topology()
if topo.node_count <= 1:
return InterleaveDecision(False, "single NUMA node; interleave not needed")
model_mib = model_size_bytes // (1024 * 1024)
largest = topo.largest_node_free_mib
total = topo.total_free_mib
if model_mib <= largest:
return InterleaveDecision(
False,
f"model ~{model_mib} MiB fits the largest node's free RAM "
f"(~{largest} MiB); keeping local placement",
)
avail = numactl_available() if has_numactl is None else has_numactl
if not avail:
# Needed but unavailable: surface it; caller decides whether to block.
return InterleaveDecision(
False,
f"model ~{model_mib} MiB exceeds the largest NUMA node's free RAM "
f"(~{largest} MiB) and needs interleaving across {topo.node_count} nodes, "
f"but `numactl` is not installed. Install numactl (e.g. `apt install "
f"numactl`) or the model may fail to fit a single node.",
)
if model_mib > total:
return InterleaveDecision(
False,
f"model ~{model_mib} MiB exceeds total free RAM across all nodes "
f"(~{total} MiB); interleave cannot help -- free memory or use a smaller quant",
)
return InterleaveDecision(
True,
f"model ~{model_mib} MiB exceeds the largest NUMA node's free RAM "
f"(~{largest} MiB) but fits across {topo.node_count} nodes (~{total} MiB total "
f"free); wrapping with numactl --interleave=all",
prefix=("numactl", "--interleave=all"),
)

119
studio/backend/tests/test_cpu_only_defaults.py Normal file
View file

@ -0,0 +1,119 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Guards for CPU-only safe llama-server defaults in load_model.
On a CPU-only host (no discrete GPU, not Apple Metal) none of the GPU/Apple fit
branches run, so the launch used to keep GPU defaults: `--flash-attn on`, `--fit on`,
and the model's full native context. For large MLA/sparse-attention/MTP GGUFs (e.g.
GLM-5.2) `--fit`'s graph-reserve estimator aborts (llama.cpp #21932) and CPU flash
attention is unsafe. These tests pin that the launch now derives `_cpu_only` and uses
it to disable --fit and default flash-attn off, while leaving GPU/Apple behaviour and
user overrides intact. Source/AST level: load_model is too entangled to drive E2E.
"""
from __future__ import annotations
import ast
import inspect
import sys
import textwrap
import types as _types
from pathlib import Path
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
try:
import structlog # noqa: F401
except ImportError:
_s = _types.ModuleType("structlog")
_s.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
_s.BoundLogger = type("BoundLogger", (), {})
sys.modules["structlog"] = _s
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
def _load_model_src() -> str:
return textwrap.dedent(inspect.getsource(LlamaCppBackend.load_model))
def test_cpu_only_flag_derived_from_no_gpu_and_not_apple():
src = _load_model_src()
assert "_cpu_only = (not gpus) and not _is_apple_silicon()" in src
def test_flash_attn_default_is_off_on_cpu_only():
"""The base cmd must use a computed flash default, off for CPU-only."""
src = _load_model_src()
assert '_flash_default = "off" if _cpu_only else "on"' in src
# And the base cmd must NOT hardcode flash-attn on anymore.
assert '"on", # Force flash attention for speed' not in src
# The --flash-attn argument in the base cmd list is the computed default.
assert "_flash_default, # CPU-only" in src
def test_fit_disabled_on_cpu_only():
src = _load_model_src()
assert "if _cpu_only and use_fit:" in src
fn = ast.parse(src).body[0]
# There is an `if _cpu_only and use_fit:` whose body sets use_fit = False.
found = False
for node in ast.walk(fn):
if (
isinstance(node, ast.If)
and isinstance(node.test, ast.BoolOp)
and any(
isinstance(v, ast.Name) and v.id == "_cpu_only"
for v in ast.walk(node.test)
)
):
for n in node.body:
if (
isinstance(n, ast.Assign)
and any(isinstance(t, ast.Name) and t.id == "use_fit" for t in n.targets)
and isinstance(n.value, ast.Constant)
and n.value.value is False
):
found = True
assert found, "expected `if _cpu_only and use_fit: ... use_fit = False`"
def test_gpu_path_still_emits_fit_on():
"""Backwards compat: the GPU branch still emits --fit on (only CPU changes)."""
src = _load_model_src()
assert 'cmd.extend(["--fit", "on"])' in src
# ---- Phase 3: CPU context cap + RAM preflight ------------------------------
def test_cpu_context_ceiling_constant_is_sane():
from core.inference import llama_cpp as m
assert hasattr(m, "_CPU_CTX_AUTO_CEILING")
# A safe chat ceiling, far below a model's million-token native context.
assert 4096 <= m._CPU_CTX_AUTO_CEILING <= 131072
assert 0.5 < m._CPU_RAM_BUDGET_FRAC <= 1.0
def test_cpu_only_caps_auto_context_but_honors_explicit():
src = _load_model_src()
# Cap only fires for an auto context (requested_ctx <= 0) over the ceiling;
# an explicit -c (requested_ctx > 0) is honored.
assert "requested_ctx <= 0 and effective_ctx > _CPU_CTX_AUTO_CEILING" in src
assert "effective_ctx = _cpu_cap" in src
def test_cpu_context_cap_reuses_fit_helper_against_ram():
"""The cap reuses _fit_context_to_vram with the system-RAM budget, not VRAM."""
src = _load_model_src()
assert "_available_system_memory_mib()" in src
assert "self._fit_context_to_vram(" in src
assert "budget_frac = _CPU_RAM_BUDGET_FRAC" in src
def test_cpu_ram_preflight_warns_when_weights_exceed_ram():
src = _load_model_src()
assert "CPU-only memory preflight" in src

106
studio/backend/tests/test_numa_interleave.py Normal file
View file

@ -0,0 +1,106 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Unit tests for the NUMA auto-interleave decision (core/inference/numa.py).
Models the user's dual-NUMA Xeon (HF screenshots #23): node 0 ~465 GB free, node 1
~223 GB free; a 583 GB GGUF exceeds the largest single node but fits across both, so
`numactl --interleave=all` is the right call. The decision is pure and topology is
injected, so these are deterministic on any host (no /sys, no numactl needed).
"""
from __future__ import annotations
import sys
from pathlib import Path
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
from core.inference.numa import ( # noqa: E402
InterleaveDecision,
NumaTopology,
_parse_online,
decide_interleave,
)
_GiB = 1024**3
_MiB = 1024**2
# The user's box, in MiB free per node (from `numactl --hardware`).
_USER_TOPO = NumaTopology(node_free_mib={0: 465594, 1: 223814})
_SINGLE = NumaTopology(node_free_mib={0: 900_000})
def test_parse_online_ranges():
assert _parse_online("0-1") == [0, 1]
assert _parse_online("0,2-3") == [0, 2, 3]
assert _parse_online("3") == [3]
assert _parse_online("") == []
def test_topology_aggregates():
assert _USER_TOPO.node_count == 2
assert _USER_TOPO.largest_node_free_mib == 465594
assert _USER_TOPO.total_free_mib == 465594 + 223814
def test_interleaves_when_model_exceeds_largest_node_but_fits_across():
# 583 GB model: > 465 GB (node 0) but < ~689 GB total.
d = decide_interleave(
583 * _GiB, cpu_only=True, topology=_USER_TOPO, has_numactl=True
)
assert d.interleave is True
assert d.prefix == ("numactl", "--interleave=all")
assert "interleave=all" in d.reason
def test_no_interleave_when_model_fits_largest_node():
# A 200 GB model fits node 0's 465 GB free -> keep local placement.
d = decide_interleave(
200 * _GiB, cpu_only=True, topology=_USER_TOPO, has_numactl=True
)
assert d.interleave is False
assert d.prefix == ()
assert "fits" in d.reason
def test_no_interleave_on_gpu_host():
d = decide_interleave(583 * _GiB, cpu_only=False, topology=_USER_TOPO, has_numactl=True)
assert d.interleave is False
assert "not cpu-only" in d.reason
def test_no_interleave_single_node():
d = decide_interleave(583 * _GiB, cpu_only=True, topology=_SINGLE, has_numactl=True)
assert d.interleave is False
assert "single NUMA node" in d.reason
def test_model_too_big_for_all_nodes_blocks_with_message():
# 800 GB > ~689 GB total free across both nodes -> interleave can't help.
d = decide_interleave(800 * _GiB, cpu_only=True, topology=_USER_TOPO, has_numactl=True)
assert d.interleave is False
assert "exceeds total free RAM" in d.reason
def test_numactl_missing_surfaces_actionable_warning():
d = decide_interleave(583 * _GiB, cpu_only=True, topology=_USER_TOPO, has_numactl=False)
assert d.interleave is False
assert "numactl` is not installed" in d.reason
def test_unknown_model_size_does_not_force_interleave():
for size in (None, 0, -1):
d = decide_interleave(size, cpu_only=True, topology=_USER_TOPO, has_numactl=True)
assert d.interleave is False
def test_decision_is_frozen_dataclass():
d = InterleaveDecision(True, "x", ("numactl", "--interleave=all"))
try:
d.interleave = False # type: ignore[misc]
except AttributeError:
return
raise AssertionError("InterleaveDecision should be immutable")

216
studio/backend/tests/test_sched_reserve_abort.py Normal file
View file

@ -0,0 +1,216 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Guards for the ggml graph-scheduler abort handling in load_model.
A CPU-only user loading GLM-5.2 (MLA + sparse-attention "indexer" + embedded MTP)
hit `GGML_ASSERT(*cur_backend_id != -1)` in `ggml_backend_sched_split_graph` during
`sched_reserve`: the CPU backend cannot run an op in the graph. Studio's startup
crash raised a generic "invalid GGUF or out of memory" message and the UI replayed
`POST /load`, re-reading the ~583 GB weights into the identical crash every ~3.5 min.
These tests pin: (1) the abort matcher recognises the real backtrace tail and only
that, (2) the classifier surfaces the actionable message, (3) the per-(binary,model)
memo round-trips and is mtime-invalidated, and (4) load_model fails fast on a memoed
abort and records on crash. No GPU, no network, fully deterministic.
"""
from __future__ import annotations
import ast
import inspect
import os
import sys
import textwrap
import types as _types
from pathlib import Path
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
# External-dep stubs so importing the backend doesn't require structlog / loggers /
# httpx -- only installed when the real module is missing (mirrors test_tp_vision_regression).
try:
import structlog # noqa: F401
except ImportError:
_structlog_stub = _types.ModuleType("structlog")
_structlog_stub.get_logger = lambda *a, **k: __import__("logging").getLogger("stub")
sys.modules["structlog"] = _structlog_stub
try:
import loggers # noqa: F401
except ImportError:
_loggers_stub = _types.ModuleType("loggers")
_loggers_stub.get_logger = lambda name: __import__("logging").getLogger(name)
sys.modules["loggers"] = _loggers_stub
try:
import httpx # noqa: F401
except ImportError:
_httpx_stub = _types.ModuleType("httpx")
for _exc in (
"ConnectError", "TimeoutException", "ReadTimeout", "ReadError",
"RemoteProtocolError", "CloseError", "HTTPError", "RequestError",
):
setattr(_httpx_stub, _exc, type(_exc, (Exception,), {}))
_httpx_stub.Timeout = type("T", (), {"__init__": lambda s, *a, **k: None})
_httpx_stub.Response = type("Response", (), {})
_httpx_stub.Client = type(
"C", (),
{"__init__": lambda s, **kw: None, "__enter__": lambda s: s, "__exit__": lambda s, *a: None},
)
sys.modules["httpx"] = _httpx_stub
from core.inference.llama_cpp import LlamaCppBackend # noqa: E402
# The real crash, faithfully reproduced from the user's llama-server log (HF
# screenshots discussion #23). The GGML_ASSERT line is followed by ~130 [New LWP]
# lines and then the gdb backtrace; the assert line scrolls out of a short tail.
_FULL_ABORT = "\n".join(
[
"0.09.350.752 W llama_context: n_ctx_seq (4096) < n_ctx_train (1048576)",
"/tmp/llama.cpp/ggml/src/ggml-backend.cpp:1242: GGML_ASSERT(*cur_backend_id != -1) failed",
]
+ [f"[New LWP {2147067 - i}]" for i in range(130)]
+ [
"[Thread debugging using libthread_db enabled]",
"#1 0x... in ggml_print_backtrace () from /tmp/llama.cpp/build/bin/libggml-base.so.0",
"#2 0x... in ggml_abort () from /tmp/llama.cpp/build/bin/libggml-base.so.0",
"#3 0x... in ggml_backend_sched_split_graph () from /tmp/llama.cpp/build/bin/libggml-base.so.0",
"#4 0x... in llama_context::graph_reserve(...) () from /tmp/llama.cpp/build/bin/libllama.so.0",
"#5 0x... in llama_context::sched_reserve() () from /tmp/llama.cpp/build/bin/libllama.so.0",
]
)
# What a 50-line tail actually contains: the GGML_ASSERT line is gone, but the
# backtrace markers (ggml_abort, ggml_backend_sched_split_graph) remain. The matcher
# must still fire on this -- that's the realistic input at the recording site.
_ABORT_TAIL = "\n".join(_FULL_ABORT.splitlines()[-50:])
# The other ggml abort Studio already handles (#6415 split-axis): must NOT be
# misclassified as a scheduler-reserve abort.
_SPLIT_AXIS_ABORT = (
"ggml/src/ggml-backend-meta.cpp:541: "
"GGML_ASSERT(src_ss[0].axis != GGML_BACKEND_SPLIT_AXIS_0) failed\n"
"#3 ggml_backend_sched_split_graph ()"
)
_OOM_OUTPUT = "llama_model_load: error loading model: unable to allocate buffer\nkilled"
_CLEAN_OUTPUT = "main: server is listening on http://127.0.0.1:8080 - starting the main loop"
# ---- matcher ---------------------------------------------------------------
def test_matcher_fires_on_full_abort_and_short_tail():
assert LlamaCppBackend._is_sched_reserve_abort(_FULL_ABORT)
# The headline guarantee: the matcher survives the [New LWP] scroll.
assert "GGML_ASSERT(*cur_backend_id != -1)".lower() not in _ABORT_TAIL.lower()
assert LlamaCppBackend._is_sched_reserve_abort(_ABORT_TAIL)
def test_matcher_ignores_unrelated_crashes():
assert not LlamaCppBackend._is_sched_reserve_abort(_SPLIT_AXIS_ABORT)
assert not LlamaCppBackend._is_sched_reserve_abort(_OOM_OUTPUT)
assert not LlamaCppBackend._is_sched_reserve_abort(_CLEAN_OUTPUT)
assert not LlamaCppBackend._is_sched_reserve_abort("")
def test_matcher_requires_both_a_ggml_marker_and_a_scheduler_marker():
# scheduler word without any ggml abort/assert -> not our abort.
assert not LlamaCppBackend._is_sched_reserve_abort("graph_reserve completed in 3ms")
# ggml abort without a scheduler marker -> some other assert, not ours.
assert not LlamaCppBackend._is_sched_reserve_abort("ggml_abort: tensor type mismatch")
# ---- classifier ------------------------------------------------------------
def test_classifier_surfaces_actionable_message():
msg = LlamaCppBackend._classify_llama_start_failure(
_ABORT_TAIL, gguf_path="/x/GLM-5.2-UD-Q6_K-00001-of-00014.gguf",
model_identifier="unsloth/GLM-5.2-GGUF", returncode=-6,
)
assert msg == LlamaCppBackend._sched_reserve_abort_message()
# The message names the real cause, not the generic invalid-GGUF/OOM fallback.
assert "ggml_backend_sched_split_graph" in msg
assert "enough memory" not in msg # i.e. not the generic fallback
def test_classifier_generic_fallback_unchanged_for_unknown_crash():
msg = LlamaCppBackend._classify_llama_start_failure(
"some unrelated failure", gguf_path=None, model_identifier=None, returncode=-6,
)
assert "failed to start" in msg and "enough memory" in msg
# ---- memo round-trip / invalidation ---------------------------------------
def test_memo_round_trip_and_isolation(tmp_path):
binary = tmp_path / "llama-server"
binary.write_text("x")
b, model = str(binary), "unsloth/GLM-5.2-GGUF"
LlamaCppBackend._sched_reserve_abort_keys.clear()
assert not LlamaCppBackend._sched_reserve_aborts(b, model)
LlamaCppBackend._record_sched_reserve_abort(b, model)
assert LlamaCppBackend._sched_reserve_aborts(b, model)
# A different model on the same binary is unaffected.
assert not LlamaCppBackend._sched_reserve_aborts(b, "unsloth/Qwen3.5-4B-MTP-GGUF")
LlamaCppBackend._sched_reserve_abort_keys.clear()
def test_memo_invalidated_by_binary_mtime_change(tmp_path):
"""A `unsloth studio update` swaps the binary -> the memo must not persist."""
binary = tmp_path / "llama-server"
binary.write_text("v1")
b, model = str(binary), "unsloth/GLM-5.2-GGUF"
LlamaCppBackend._sched_reserve_abort_keys.clear()
LlamaCppBackend._record_sched_reserve_abort(b, model)
assert LlamaCppBackend._sched_reserve_aborts(b, model)
# Bump mtime to a distinct ns value (simulate a rebuilt binary).
st = binary.stat()
os.utime(binary, ns=(st.st_atime_ns + 10**9, st.st_mtime_ns + 10**9))
assert not LlamaCppBackend._sched_reserve_aborts(b, model)
LlamaCppBackend._sched_reserve_abort_keys.clear()
def test_memo_safe_with_missing_binary_or_model():
# None binary/model -> no key -> never aborts, never raises.
assert not LlamaCppBackend._sched_reserve_aborts(None, "m")
assert not LlamaCppBackend._sched_reserve_aborts("/x", None)
LlamaCppBackend._record_sched_reserve_abort(None, None) # no-op, no raise
# ---- load_model wiring (source-level, no GPU/network needed) ---------------
def _load_model_src() -> str:
return textwrap.dedent(inspect.getsource(LlamaCppBackend.load_model))
def test_load_model_fails_fast_on_memoed_abort():
"""load_model must consult the memo and raise the actionable message before the
download/spawn, so a replayed /load doesn't re-read the weights."""
src = _load_model_src()
assert "_sched_reserve_aborts(binary, model_identifier)" in src
assert "_sched_reserve_abort_message()" in src
# The guard must precede the model download (the expensive reload it prevents).
fn = ast.parse(src).body[0]
guard_line = next(
node.lineno for node in ast.walk(fn)
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
and node.func.attr == "_sched_reserve_aborts"
)
download_line = next(
(node.lineno for node in ast.walk(fn)
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
and node.func.attr == "_download_gguf"),
None,
)
assert download_line is None or guard_line < download_line
def test_load_model_records_abort_on_crash():
"""On a startup crash matching the signature, load_model must record the memo."""
src = _load_model_src()
assert "_record_sched_reserve_abort(binary, model_identifier)" in src
assert "_is_sched_reserve_abort(" in src