diff --git a/.gitignore b/.gitignore index a839633790..9f7d4b8c60 100644 --- a/.gitignore +++ b/.gitignore @@ -235,3 +235,5 @@ package-lock.json !studio/backend/core/data_recipe/oxc-validator/package-lock.json !studio/package-lock.json llama.cpp/ +# Stray "~" dir some tools create from a literal ~ TMPDIR; never part of the repo. +/~/ diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 16a401d115..92cc2c748c 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1147,9 +1147,9 @@ class LlamaCppBackend: """Manages a llama-server subprocess for GGUF model inference. Lifecycle: - 1. load_model() — start llama-server with the GGUF file - 2. generate_chat_completion() — proxy to /v1/chat/completions, stream back - 3. unload_model() — terminate the subprocess + 1. load_model(): start llama-server with the GGUF file + 2. generate_chat_completion(): proxy to /v1/chat/completions, stream back + 3. unload_model(): terminate the subprocess """ def __init__(self): @@ -1879,11 +1879,12 @@ class LlamaCppBackend: return total @staticmethod - def _amd_apu_wants_unified_memory() -> bool: + def _amd_apu_wants_unified_memory(gpu_indices = None) -> bool: """True only for AMD unified-memory APUs (gfx1150/gfx1151), where - GGML_CUDA_ENABLE_UNIFIED_MEMORY lets llama.cpp use shared system RAM. - False elsewhere (the env hurts discrete GPUs). ROCm reuses torch.cuda.*; - gcnArchName suffix is stripped.""" + GGML_CUDA_ENABLE_UNIFIED_MEMORY lets llama.cpp use shared system RAM (it + hurts discrete GPUs). gpu_indices (PHYSICAL ids) scopes the check to the + selected GPUs, so a dGPU on a mixed host is not treated as unified-memory; + None means every visible GPU.""" try: import torch @@ -1891,12 +1892,39 @@ class LlamaCppBackend: return False if not (hasattr(torch, "cuda") and torch.cuda.is_available()): return False - for _i in range(torch.cuda.device_count()): + # Map visible ordinal -> physical id via the active ROCm mask (HIP, + # then ROCR, then CUDA), mirroring _get_gpu_memory's ROCm branch. + physical_ids: Optional[list[int]] = None + hip_v = os.environ.get("HIP_VISIBLE_DEVICES") + rocr_v = os.environ.get("ROCR_VISIBLE_DEVICES") + cvd = ( + hip_v + if hip_v is not None + else rocr_v + if rocr_v is not None + else os.environ.get("CUDA_VISIBLE_DEVICES") + ) + if cvd is not None: try: - _arch = getattr(torch.cuda.get_device_properties(_i), "gcnArchName", "") or "" + physical_ids = [int(x.strip()) for x in cvd.split(",") if x.strip()] + except ValueError: + physical_ids = None + arch_by_id: dict[int, str] = {} + for ordinal in range(torch.cuda.device_count()): + try: + _arch = ( + getattr(torch.cuda.get_device_properties(ordinal), "gcnArchName", "") or "" + ) except Exception: continue - if _arch.split(":")[0].strip().lower() in {"gfx1150", "gfx1151"}: + pid = ( + physical_ids[ordinal] + if physical_ids is not None and ordinal < len(physical_ids) + else ordinal + ) + arch_by_id[pid] = _arch.split(":")[0].strip().lower() + for _i in list(gpu_indices) if gpu_indices is not None else list(arch_by_id): + if arch_by_id.get(_i) in {"gfx1150", "gfx1151"}: return True except Exception: return False @@ -2130,6 +2158,48 @@ class LlamaCppBackend: logger.debug(f"torch GPU probe failed: {e}") return [] + @staticmethod + def _available_system_memory_mib() -> Optional[int]: + """Available system RAM in MiB (psutil, then /proc/meminfo), or None if + neither is readable. On a unified-memory APU this, not the ROCm-reported + VRAM, is the real ceiling: the weights load into shared system RAM.""" + try: + import psutil + return int(psutil.virtual_memory().available // (1024 * 1024)) + except Exception: + pass + try: + with open("/proc/meminfo") as f: + for line in f: + if line.startswith("MemAvailable:"): + return int(line.split()[1]) // 1024 # kB -> MiB + except Exception: + pass + return None + + @staticmethod + def _apu_ram_shortfall_message( + model_size_bytes: int, + avail_mib: Optional[int], + headroom_mib: int = 2048, + ) -> Optional[str]: + """On a unified-memory APU, return a user-facing refusal when the weights + cannot fit in available system RAM (else None). Weights only: KV/context + auto-reduce, so counting them too would refuse loads that would succeed. + None avail (unknown RAM) never refuses.""" + if avail_mib is None: + return None + need_mib = model_size_bytes / (1024 * 1024) + if need_mib <= avail_mib - headroom_mib: + return None + return ( + f"This model needs about {need_mib / 1024:.0f} GB but only about " + f"{avail_mib / 1024:.0f} GB of memory is available. On a unified-memory " + "APU the weights load into system RAM, so a larger model is stopped by " + "the OS mid-load. Use a smaller or more quantized GGUF, or free memory " + "(on WSL, raise the memory limit in .wslconfig)." + ) + # Skip the wait when the last kill is older than this; the driver has # already reclaimed the prior process's allocations. _VRAM_SETTLE_WINDOW_S: float = 15.0 @@ -3814,7 +3884,10 @@ class LlamaCppBackend: @staticmethod def _classify_llama_start_failure( - output: str, gguf_path: Optional[str], model_identifier: Optional[str] + output: str, + gguf_path: Optional[str], + model_identifier: Optional[str], + returncode: Optional[int] = None, ) -> str: """Explain *why* llama-server failed to start, from its output. @@ -3886,6 +3959,24 @@ class LlamaCppBackend: "Ollama instead." ) + # SIGKILL with no diagnostic output is the OOM killer (e.g. a model too + # large for the WSL VM's RAM cap); name it actionably. + if returncode == -9: + return ( + "llama-server was stopped by the operating system (signal 9), " + "most likely out of memory. Try a smaller or more quantized " + "GGUF, lower the context length, or free memory (on WSL, raise " + "the memory limit in .wslconfig)." + ) + # SIGTERM is also how an unload/cancel or a supervisor stops the server, + # so report it neutrally rather than blaming memory. + if returncode == -15: + return ( + "llama-server was terminated (signal 15) before it became " + "healthy. If you cancelled or unloaded the model this is " + "expected; otherwise check the llama-server log for the cause." + ) + # Fallback: genuinely unknown failure (OOM, missing binary ...). return ( "llama-server failed to start. " @@ -4064,6 +4155,71 @@ class LlamaCppBackend: and ("unknown" in text or "unsupported" in text or "not supported" in text) ) + @staticmethod + def _output_has_nonprojector_diagnostic(output: str) -> bool: + """True when the output already names a concrete non-projector cause (out + of memory, an unsupported architecture, a tensor-parallel limit). A hard + crash carrying such a marker must surface that error, not be silently + retried text-only as if the vision projector were at fault; a bare crash + with no marker still gets the text-only retry. + """ + text = (output or "").lower() + return any( + m in text + for m in ( + "out of memory", + "failed to allocate", + "unknown model architecture", + "split_mode_tensor not implemented", + ) + ) + + @staticmethod + def _is_signal_crash(returncode: Optional[int]) -> bool: + """True only on a hard fault (SIGSEGV/SIGABRT/SIGILL/SIGFPE/SIGBUS or a + Windows 0xC0000000+ status), not SIGKILL/SIGTERM/SIGINT (OOM killer / + unload) nor a clean exit or still-running (None) process. + """ + if returncode is None: + return False + if returncode >= 0xC0000000: # Windows access violation / illegal instruction + return True + return -returncode in (4, 6, 7, 8, 11) # SIGILL SIGABRT SIGBUS SIGFPE SIGSEGV + + @staticmethod + def _with_flash_attn_off(cmd: list[str]) -> Optional[list[str]]: + """Return cmd with flash attention forced off, or None when its effective + (last-wins) value is already off/absent so there is nothing to retry. FA + kernels hard-crash at startup on some ROCm builds; disabling FA keeps + vision and MTP, the least destructive rung. A bare --flash-attn/-fa reads + as on, so it counts toward the effective value and is neutralised too; + every form is flipped in place (length preserved for downstream slices).""" + out = list(cmd) + + def explicit(i): + nxt = out[i + 1] if i + 1 < len(out) else None + return nxt if nxt in ("on", "auto", "off") else None + + effective = None + for i, tok in enumerate(out): + if tok.startswith(("--flash-attn=", "-fa=")): + effective = tok.partition("=")[2] + elif tok in ("--flash-attn", "-fa"): + effective = explicit(i) or "on" + if effective not in ("on", "auto"): + return None + for i, tok in enumerate(out): + if tok.startswith(("--flash-attn=", "-fa=")): + flag, _, value = tok.partition("=") + if value in ("on", "auto"): + out[i] = f"{flag}=off" + elif tok in ("--flash-attn", "-fa"): + if explicit(i) in ("on", "auto"): + out[i + 1] = "off" + elif explicit(i) is None: # bare flag (reads as on) -> explicit off + out[i] = f"{tok}=off" + return out + @staticmethod def _strip_mmproj_args(cmd: list[str]) -> list[str]: """Return cmd without the '--mmproj ' pair (text-only retry). @@ -4488,6 +4644,7 @@ class LlamaCppBackend: "Vision-capable GGUF loaded without a usable mmproj; " "image input will be disabled for this session" ) + model_size = None # set in the fit try; used by the APU RAM guard try: gguf_size = self._get_gguf_size_bytes(model_path) # Include GPU-loaded mmproj in the fit budget (#5825). @@ -5069,6 +5226,17 @@ class LlamaCppBackend: tp_tensor_split = None effective_ctx = requested_ctx # fall back to original + # Unified-memory APUs load weights into system RAM (under WSL the VM + # cap, not the ROCm-reported VRAM, is the real ceiling); refuse an + # oversize load the OS would otherwise kill mid-flight. Base model + # only: an optional MTP drafter is dropped by the MTP-drop fallback. + if model_size is not None and self._amd_apu_wants_unified_memory(gpu_indices): + _ram_msg = self._apu_ram_shortfall_message( + model_size, self._available_system_memory_mib() + ) + if _ram_msg: + raise RuntimeError(_ram_msg) + # Audio input straight from the mmproj (clip.has_audio_encoder), # independent of token names. self._mmproj_has_audio = False @@ -5368,7 +5536,7 @@ class LlamaCppBackend: # AMD unified-memory APUs (gfx1150/gfx1151): let llama.cpp use # shared system RAM. setdefault so a user value wins. - if self._amd_apu_wants_unified_memory(): + if self._amd_apu_wants_unified_memory(gpu_indices): env.setdefault("GGML_CUDA_ENABLE_UNIFIED_MEMORY", "1") logger.info("AMD unified-memory APU: set GGML_CUDA_ENABLE_UNIFIED_MEMORY=1") @@ -5419,6 +5587,9 @@ class LlamaCppBackend: # retry once with --fit off before declaring the load failed. # Never retry when fit was requested (use_fit) or the caller # passed an explicit fit flag via extra args. + # Argv actually launched (post --fit off / MTP); text-only retry strips this. + _last_spawn_cmd = list(cmd) + def _spawn_and_wait(run_cmd, *, label = ""): """Start llama-server with run_cmd and wait for health. @@ -5426,6 +5597,7 @@ class LlamaCppBackend: crashes during startup and run_cmd is eligible (see _fit_off_retry_eligible). """ + nonlocal _last_spawn_cmd _fit_retry_allowed = self._fit_off_retry_eligible(run_cmd, use_fit) for _spawn_attempt in (0, 1): # Defensive kill: drop an orphan Popen a concurrent load may @@ -5460,6 +5632,7 @@ class LlamaCppBackend: # Best-effort; never block the load on logging. logger.debug(f"Could not open llama-server log file: {e}") self._llama_log_path = None + _last_spawn_cmd = list(run_cmd) self._process = subprocess.Popen( run_cmd, stdout = subprocess.PIPE, @@ -5527,6 +5700,28 @@ class LlamaCppBackend: ) healthy = _spawn_and_wait(cmd) + # Flash-attention kernels hard-crash at startup on some ROCm/GPU + # builds (frequently inside the vision tower). Disabling FA keeps + # both vision and MTP, so retry that way before dropping either. + # Only on a hard fault with FA on; a cancel/unload stops respawn. + if not healthy and not self._cancel_event.is_set(): + _fa_rc = self._process.poll() if self._process is not None else None + _fa_cmd = ( + self._with_flash_attn_off(_last_spawn_cmd) + if self._is_signal_crash(_fa_rc) + else None + ) + if _fa_cmd is not None: + logger.warning( + "llama-server hard-crashed at startup (exit %s) with " + "flash attention on; retrying once with --flash-attn " + "off (keeps vision and MTP).", + _fa_rc, + ) + self._kill_process() + cmd = _fa_cmd + healthy = _spawn_and_wait(_fa_cmd, label = "-noflash") + # MTP from Studio's spec flags or the user's (extra_args # --spec-type / LLAMA_ARG_SPEC_TYPE). The env reaches the child # only when neither emits a spec flag, so consult it only then. @@ -5553,11 +5748,32 @@ class LlamaCppBackend: and not self._cancel_event.is_set() and not self._probe_mtp_decode() ): - logger.warning( - "MTP speculative decoding crashed on the first decode " - "under tensor parallelism; retrying without it." + # A first-decode hard fault is usually the FA kernel: retry + # FA-off (keeps MTP) before dropping speculative decoding below. + _probe_rc = self._process.poll() if self._process is not None else None + _fa_cmd = ( + self._with_flash_attn_off(_last_spawn_cmd) + if self._is_signal_crash(_probe_rc) + else None ) healthy = False + if _fa_cmd is not None: + logger.warning( + "MTP first-decode hard-crashed (exit %s) with flash " + "attention on; retrying with --flash-attn off.", + _probe_rc, + ) + self._kill_process() + cmd = _fa_cmd + healthy = ( + _spawn_and_wait(_fa_cmd, label = "-noflash-mtp") + and self._probe_mtp_decode() + ) + if not healthy: + logger.warning( + "MTP speculative decoding crashed on the first decode " + "under tensor parallelism; retrying without it." + ) # Any MTP request can abort the server: a separate drafter # (Gemma) on a binary that predates its arch, or an embedded # head (Qwen) the binary cannot build. Retry once with the @@ -5621,25 +5837,39 @@ class LlamaCppBackend: self._speculative_type = "default" _mtp_active_for_launched_server = False - # A vision GGUF launched with --mmproj can abort when the - # installed llama.cpp is too old for the model's projector - # ("Unknown projector type"); in that one case retry once - # text-only rather than failing the whole load. + # A too-old llama.cpp can reject a model's --mmproj projector + # (format message or a bare SIGSEGV); retry once text-only. if not healthy: out = "\n".join(self._stdout_lines[-50:]) + # Read the crash code before _kill_process() clears _process. + _crash_rc = self._process.poll() if self._process is not None else None self._kill_process() - if launched_with_mmproj and self._is_projector_incompatibility(out): + # Skip if a cancel/unload is pending (mirrors the MTP guard). + if ( + launched_with_mmproj + and not self._cancel_event.is_set() + and ( + self._is_projector_incompatibility(out) + or ( + self._is_signal_crash(_crash_rc) + and not self._output_has_nonprojector_diagnostic(out) + ) + ) + ): logger.warning( "llama-server could not load this model's vision " "projector (--mmproj). The installed llama.cpp build is " "likely too old for it. Loading text-only for this " "session; run 'unsloth studio update' to enable vision." ) - cmd = self._strip_mmproj_args(cmd) + cmd = self._strip_mmproj_args(_last_spawn_cmd) self._is_vision = False self._mmproj_has_audio = False self._start_llama_process(cmd, env) if not self._wait_for_health(timeout = 600.0): + # Read the exit code before _kill_process() clears it, so + # an OS-killed text-only retry still gets the OOM message. + _retry_rc = self._process.poll() if self._process is not None else None self._kill_process() raise RuntimeError( "Vision projector incompatible with this llama.cpp " @@ -5648,6 +5878,7 @@ class LlamaCppBackend: "\n".join(self._stdout_lines[-50:]), gguf_path, self._model_identifier, + _retry_rc, ) ) else: @@ -5656,6 +5887,7 @@ class LlamaCppBackend: out, gguf_path, self._model_identifier, + _crash_rc, ) ) @@ -8411,7 +8643,7 @@ class LlamaCppBackend: try: # llama-server's /apply-template renders tool declarations # into the prompt when ``tools`` is supplied, so pass them - # through — otherwise tool-schema tokens go uncounted. + # through, otherwise tool-schema tokens go uncounted. template_body = {"messages": template_messages} if tools: template_body["tools"] = tools diff --git a/studio/backend/tests/test_amd_apu_unified_memory.py b/studio/backend/tests/test_amd_apu_unified_memory.py index 104182a232..4df9e85b30 100644 --- a/studio/backend/tests/test_amd_apu_unified_memory.py +++ b/studio/backend/tests/test_amd_apu_unified_memory.py @@ -47,6 +47,30 @@ def test_apu_unified_memory_gating(monkeypatch, hip, archs, expected): assert LlamaCppBackend._amd_apu_wants_unified_memory() is expected +def test_apu_guard_scopes_to_selected_gpu(monkeypatch): + # Mixed host: physical id 0 = discrete gfx1100, 1 = gfx1151 APU. + for _m in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"): + monkeypatch.delenv(_m, raising = False) + monkeypatch.setitem(sys.modules, "torch", _fake_torch("6.2.0", ["gfx1100", "gfx1151"])) + # Selecting only the dGPU, or an empty selection, must not be unified-memory. + assert LlamaCppBackend._amd_apu_wants_unified_memory([0]) is False + assert LlamaCppBackend._amd_apu_wants_unified_memory([]) is False + # Selecting the APU, or no selection, does. + assert LlamaCppBackend._amd_apu_wants_unified_memory([1]) is True + assert LlamaCppBackend._amd_apu_wants_unified_memory() is True + + +def test_apu_guard_honors_hip_visible_devices_mask(monkeypatch): + # ROCm resolves ids via HIP first: the mask exposes only the APU as ordinal 0 + # but physical id 1, so the selection [1] must still match. + monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False) + monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False) + monkeypatch.setenv("HIP_VISIBLE_DEVICES", "1") + monkeypatch.setitem(sys.modules, "torch", _fake_torch("6.2.0", ["gfx1151"])) + assert LlamaCppBackend._amd_apu_wants_unified_memory([1]) is True + assert LlamaCppBackend._amd_apu_wants_unified_memory([0]) is False + + def test_cpu_no_cuda_returns_false(monkeypatch): monkeypatch.setitem(sys.modules, "torch", _fake_torch("6.2.0", [], cuda_ok = False)) assert LlamaCppBackend._amd_apu_wants_unified_memory() is False @@ -55,3 +79,39 @@ def test_cpu_no_cuda_returns_false(monkeypatch): def test_missing_torch_returns_false(monkeypatch): monkeypatch.setitem(sys.modules, "torch", None) assert LlamaCppBackend._amd_apu_wants_unified_memory() is False + + +_GB = 1024**3 +_MIB_PER_GB = 1024 +# Module-level (not a class attr) so it stays a plain function, not a bound method. +_shortfall = LlamaCppBackend._apu_ram_shortfall_message + + +class TestApuRamShortfall: + """On a unified-memory APU the weights load into system RAM, so a model + larger than available RAM (the field case: a 64.6 GB GGUF on a WSL VM capped + well below the ROCm-reported APU budget) must be refused before spawning, + not left to OOM-kill the Studio process.""" + + def test_field_case_wsl_cap_refuses(self): + # 64.6 GB weights, ~46 GB available (WSL VM): refuse with guidance. + msg = _shortfall(int(64.6 * _GB), 46 * _MIB_PER_GB) + assert msg is not None + assert "65 GB" in msg and "46 GB" in msg + assert ".wslconfig" in msg + + def test_bare_metal_fits_allows(self): + # Same model, ~92 GB available (no WSL cap): allow. + assert _shortfall(int(64.6 * _GB), 92 * _MIB_PER_GB) is None + + def test_unknown_available_never_refuses(self): + assert _shortfall(int(64.6 * _GB), None) is None + + def test_boundary_at_headroom(self): + # 20 GB weights, headroom 2 GB. avail 23 GB -> fits; 21 GB -> refuse. + assert _shortfall(20 * _GB, 23 * _MIB_PER_GB) is None + assert _shortfall(20 * _GB, 21 * _MIB_PER_GB) is not None + + def test_available_system_memory_is_int_or_none(self): + v = LlamaCppBackend._available_system_memory_mib() + assert v is None or (isinstance(v, int) and v > 0) diff --git a/studio/backend/tests/test_llama_cpp_mmproj_fallback.py b/studio/backend/tests/test_llama_cpp_mmproj_fallback.py index 6ef94545fe..04d4aac9e1 100644 --- a/studio/backend/tests/test_llama_cpp_mmproj_fallback.py +++ b/studio/backend/tests/test_llama_cpp_mmproj_fallback.py @@ -40,6 +40,9 @@ from core.inference.llama_cpp import LlamaCppBackend # noqa: E402 _detect = LlamaCppBackend._is_projector_incompatibility _strip = LlamaCppBackend._strip_mmproj_args +_signal_crash = LlamaCppBackend._is_signal_crash +_flash_off = LlamaCppBackend._with_flash_attn_off +_nonproj = LlamaCppBackend._output_has_nonprojector_diagnostic # Real abort captured loading gemma-4 on a 3-day-old prebuilt (build b9496). _GEMMA4_OLD_LLAMACPP_OUT = ( @@ -103,6 +106,20 @@ class TestProjectorIncompatibilityDetector: assert _detect(out) is False +class TestSignalCrashDetector: + """_is_signal_crash flags a hard fault (SIGSEGV/SIGABRT/SIGILL/SIGFPE/SIGBUS + or a Windows 0xC0000000+ fault); not a clean exit, hung (None), or an + external kill (SIGKILL/SIGTERM/SIGINT) that an OOM/unload would cause.""" + + @pytest.mark.parametrize("rc", [-11, -6, -4, -7, -8, 0xC0000005, 0xC000001D]) + def test_program_faults_are_hard_crashes(self, rc): + assert _signal_crash(rc) is True + + @pytest.mark.parametrize("rc", [0, 1, 2, 137, None, -9, -15, -2]) + def test_clean_hung_or_external_kill_is_not(self, rc): + assert _signal_crash(rc) is False + + # A realistic vision launch argv (mirrors the live "Starting llama-server" # command), projector pair at the end. _VISION_CMD = [ @@ -170,6 +187,99 @@ class TestStripMmprojArgs: assert cmd[-1] == "/p/mm.gguf" # input untouched +class TestFlashAttnOff: + """_with_flash_attn_off is the least-destructive recovery rung: flip + '--flash-attn on' to 'off' (keeps vision + MTP), or None when there is + nothing to disable.""" + + def test_flips_on_to_off_keeping_vision_and_mtp(self): + out = _flash_off(_VISION_CMD) + assert out is not None + # FA disabled, every other capability (mmproj, MTP, ctx) preserved. + i = out.index("--flash-attn") + assert out[i + 1] == "off" + assert "--mmproj" in out and "--spec-default" in out + assert len(out) == len(_VISION_CMD) + + def test_none_when_already_off(self): + assert _flash_off(["llama-server", "--flash-attn", "off", "-c", "4096"]) is None + + def test_none_when_no_flash_attn(self): + assert _flash_off(["llama-server", "-m", "/m.gguf", "-c", "4096"]) is None + + def test_returns_new_list_input_untouched(self): + cmd = ["llama-server", "--flash-attn", "on"] + out = _flash_off(cmd) + assert out == ["llama-server", "--flash-attn", "off"] + assert cmd[-1] == "on" # input not mutated + + def test_flips_equals_form(self): + out = _flash_off(["llama-server", "--flash-attn=on", "-c", "4096"]) + assert out == ["llama-server", "--flash-attn=off", "-c", "4096"] + + def test_flips_fa_alias_and_auto(self): + assert _flash_off(["llama-server", "-fa", "auto"]) == ["llama-server", "-fa", "off"] + assert _flash_off(["llama-server", "-fa=on"]) == ["llama-server", "-fa=off"] + + def test_flips_every_occurrence_last_wins(self): + # extra_args can re-enable FA after Studio's flag; llama.cpp is last-wins, + # so one leftover 'on' would re-crash the retry. Every enable must flip. + cmd = ["llama-server", "--flash-attn", "on", "--mmproj", "/p", "--flash-attn", "on"] + out = _flash_off(cmd) + assert out is not None + assert "on" not in out + assert out.count("off") == 2 + + def test_none_when_equals_off(self): + assert _flash_off(["llama-server", "--flash-attn=off"]) is None + + def test_none_when_user_off_wins_last(self): + # User appended 'off' after Studio's 'on'; effective (last-wins) is off, + # so there is nothing to retry. + assert _flash_off(["llama-server", "--flash-attn", "on", "--flash-attn", "off"]) is None + + def test_neutralizes_trailing_bare_flag(self): + # A bare --flash-attn reads as on under last-wins; it must be neutralized + # too, else the retry re-enables FA and re-crashes. + out = _flash_off(["llama-server", "--flash-attn", "on", "--flash-attn"]) + assert out == ["llama-server", "--flash-attn", "off", "--flash-attn=off"] + assert "on" not in out + + def test_bare_flag_only(self): + assert _flash_off(["llama-server", "--flash-attn"]) == ["llama-server", "--flash-attn=off"] + assert _flash_off(["llama-server", "-fa"]) == ["llama-server", "-fa=off"] + + +class TestNonProjectorDiagnostic: + """_output_has_nonprojector_diagnostic gates the signal-only text-only retry: + a hard crash that already names OOM / a bad arch / a TP limit must surface + that error, not be silently downgraded to a non-vision session.""" + + @pytest.mark.parametrize( + "out", + [ + _OOM_OUT, + _BAD_ARCH_OUT, + "ggml_backend_cuda_buffer_type_alloc: failed to allocate buffer", + "split_mode_tensor not implemented for this architecture", + ], + ) + def test_known_nonprojector_causes_match(self, out): + assert _nonproj(out) is True + + @pytest.mark.parametrize( + "out", + [ + "", # bare crash: no marker -> still eligible for the text-only retry + _GEMMA4_OLD_LLAMACPP_OUT, # a real projector abort must NOT be suppressed + _HEALTHY_VISION_OUT, + _PORT_OUT, + ], + ) + def test_bare_or_projector_output_does_not_match(self, out): + assert _nonproj(out) is False + + class TestRetryContract: """The two helpers compose into the load_model retry decision.""" @@ -185,3 +295,43 @@ class TestRetryContract: # An OOM with --mmproj present must NOT be treated as a projector # problem: load_model errors out instead of dropping vision. assert _detect(_OOM_OUT) is False + + def test_bare_segfault_with_mmproj_yields_text_only_retry(self): + # Field report: a -11 SIGSEGV on --mmproj has no projector line; the + # signal path fires only when no other diagnostic explains the crash. + out = "" # a SIGSEGV produced no projector-format line + assert _detect(out) is False + should_retry = _detect(out) or (_signal_crash(-11) and not _nonproj(out)) + assert should_retry is True + retry_cmd = _strip(_VISION_CMD) + assert "--mmproj" not in retry_cmd and "-m" in retry_cmd + + def test_signal_crash_with_oom_output_keeps_the_real_error(self): + # A hard fault that already printed an OOM must surface it, not silently + # drop --mmproj and tell the user to update llama.cpp. + assert _signal_crash(-6) is True + should_retry = _detect(_OOM_OUT) or (_signal_crash(-6) and not _nonproj(_OOM_OUT)) + assert should_retry is False + + def test_signal_crash_with_bad_arch_does_not_drop_vision(self): + should_retry = _detect(_BAD_ARCH_OUT) or (_signal_crash(-6) and not _nonproj(_BAD_ARCH_OUT)) + assert should_retry is False + + def test_clean_nonzero_exit_with_mmproj_does_not_retry(self): + # Clean non-zero exit (bad path, port bind) is not a hard crash; stay message-based. + assert (_detect(_MISSING_OUT) or _signal_crash(1)) is False + + def test_signal_crash_tries_flash_attn_off_before_dropping_vision(self): + # Ladder order: a hard fault retries FA-off FIRST (keeps vision + MTP); + # only if THAT is None/fails do we fall back to stripping --mmproj. + assert _signal_crash(-11) is True + fa_retry = _flash_off(_VISION_CMD) + assert fa_retry is not None + assert "--mmproj" in fa_retry # vision preserved at this rung + # Last resort still available and strictly more destructive. + text_only = _strip(fa_retry) + assert "--mmproj" not in text_only + + def test_external_kill_skips_flash_attn_retry(self): + # SIGKILL (-9, OOM killer) is not a program fault: no FA-off retry. + assert _signal_crash(-9) is False diff --git a/studio/backend/tests/test_llama_cpp_start_failure_classification.py b/studio/backend/tests/test_llama_cpp_start_failure_classification.py index 8d88a61ae3..6b26121cf8 100644 --- a/studio/backend/tests/test_llama_cpp_start_failure_classification.py +++ b/studio/backend/tests/test_llama_cpp_start_failure_classification.py @@ -139,3 +139,35 @@ class TestOllamaAndFallback: def test_empty_output_is_safe(self): msg = _classify("", None, None) assert "llama-server failed to start" in msg + + +class TestOsKillReturncode: + """SIGKILL (-9) with no diagnostic output is the OOM killer and gets a named, + actionable message; SIGTERM (-15) is also unload/cancel/supervisor stop, so it + stays neutral; a recognized output still wins; a hard fault (-11) keeps the + generic fallback.""" + + def test_sigkill_with_no_output_names_oom(self): + msg = _classify("", "/models/big-bf16.gguf", "local/big", -9) + assert "signal 9" in msg + assert "out of memory" in msg.lower() + assert ".wslconfig" in msg + assert "GGUF file is valid" not in msg + + def test_sigterm_is_neutral_not_oom(self): + msg = _classify("", "/models/big-bf16.gguf", "local/big", -15) + assert "signal 15" in msg + assert "terminated" in msg.lower() + assert "out of memory" not in msg.lower() + + def test_specific_output_wins_over_os_kill_code(self): + msg = _classify(_QWEN_IMAGE_OUT, "/models/qwen-image.gguf", "local/qwen-image", -9) + assert "diffusion" in msg.lower() + assert "out of memory" not in msg.lower() + + def test_signal_crash_code_keeps_generic_message(self): + # -11 is handled by the retry ladder; if it reaches here with no output + # it gets the generic fallback, not the OOM message. + msg = _classify("", "/models/x.gguf", "local/x", -11) + assert "GGUF file is valid" in msg + assert "out of memory" not in msg.lower() diff --git a/studio/backend/tests/test_tensor_parallel.py b/studio/backend/tests/test_tensor_parallel.py index 4ddd0224a2..5b1138ef09 100644 --- a/studio/backend/tests/test_tensor_parallel.py +++ b/studio/backend/tests/test_tensor_parallel.py @@ -270,16 +270,18 @@ def test_proportional_tensor_split_is_emitted_in_tensor_mode(): def test_mtp_decode_probe_wired_under_tensor_parallel(): # MTP-draft can pass /health and crash the CUDA FA kernel only on the first # decode under --split-mode tensor. Rather than statically banning MTP+TP - # (which a future llama.cpp may support), load_model probes a decode and - # routes a failure into the existing MTP-drop fallback. + # (which a future llama.cpp may support), load_model probes a decode; a hard + # fault retries --flash-attn off first, else routes into the MTP-drop fallback. src = _load_model_source() probe = src.find("_probe_mtp_decode()") assert probe != -1, "load_model must decode-probe MTP under tensor parallelism" # Gated on tensor mode AND an MTP request (ordinary MTP loads stay unprobed). guard = src[max(0, probe - 400) : probe] assert "self._tensor_parallel" in guard and "_spec_requested_mtp" in guard - # A failed probe flips healthy so the shared MTP-drop fallback fires. - assert "healthy = False" in src[probe : probe + 400] + # A hard fault retries FA-off (keeps MTP) before flipping healthy so the + # shared MTP-drop fallback fires. + after = src[probe : probe + 900] + assert "_with_flash_attn_off" in after and "healthy = False" in after fallback = src.find("if not healthy and _spec_requested_mtp") assert 0 <= probe < fallback, "the probe must precede the MTP-drop fallback"