diff --git a/studio/backend/core/inference/sd_cpp_server.py b/studio/backend/core/inference/sd_cpp_server.py index a7c6061de0..33ce886b65 100644 --- a/studio/backend/core/inference/sd_cpp_server.py +++ b/studio/backend/core/inference/sd_cpp_server.py @@ -72,6 +72,34 @@ _TERMINAL_OK = "completed" _TERMINAL_FAIL = "failed" _TERMINAL_CANCELLED = "cancelled" +# Lines worth keeping from a dead server's output whatever their position. +_DIAGNOSTIC_MARKERS = ( + "error", + "abort", + "assert", + "unsupported", + "not implemented", + "out of memory", + "failed", + "exception", +) + + +def _diagnostic_tail(lines, *, keep: int = 20, limit: int = 1500) -> str: + """The most useful part of the captured output, not merely its last lines. + + A native abort prints its REASON first and then a long backtrace, so taking the last N lines + reported nothing but stack frames: a Metal host that died on an unimplemented op showed twenty + addresses and no cause. Marked lines come first (in order), then the last few lines for + context, deduplicated.""" + captured = list(lines) + marked = [line for line in captured if any(m in line.lower() for m in _DIAGNOSTIC_MARKERS)] + chosen: list[str] = [] + for line in marked[-keep:] + captured[-max(keep // 2, 4):]: + if line not in chosen: + chosen.append(line) + return "\n".join(chosen)[:limit] + # Grace for the best-effort native cancel to show in job status before abandoning the poll; # without the cap a lost cancel would hold the generate lock until the job ends. _CANCEL_GRACE_S = 5.0 @@ -212,7 +240,7 @@ class SdCppServer: self._dispose() raise RuntimeError(f"failed to spawn sd-server: {self._spawn_error}") if not self._wait_ready(startup_timeout): - tail = "\n".join(list(self._tail)[-30:]) + tail = _diagnostic_tail(self._tail, keep = 30) aborted = self._abort.is_set() self._kill_locked() self._dispose() @@ -467,7 +495,7 @@ class SdCppServer: return out def _died_message(self, where: str, exc: Optional[Exception]) -> str: - tail = "\n".join(list(self._tail)[-20:]) + tail = _diagnostic_tail(self._tail) base = f"sd-server connection lost during {where}" if not self.is_alive(): code = None if self._process is None else self._process.returncode diff --git a/studio/backend/tests/test_sd_cpp_server.py b/studio/backend/tests/test_sd_cpp_server.py index f74339fbf0..da3df18f8c 100644 --- a/studio/backend/tests/test_sd_cpp_server.py +++ b/studio/backend/tests/test_sd_cpp_server.py @@ -415,3 +415,34 @@ def test_start_aborted_by_concurrent_stop(patched): threading.Thread(target = _stop_soon, daemon = True).start() with pytest.raises(SdCppCancelled): s.start(_FILES, startup_timeout = 30.0) + + +def test_diagnostic_tail_keeps_the_reason_not_just_the_backtrace(): + """What a Metal host produces: the abort prints its cause, then ggml_print_backtrace fills the + buffer with stack frames. Taking the last N lines reported addresses and no cause, so the + failure was undiagnosable from the message alone.""" + lines = [ + "loading model from flux-2-klein-4b-Q2_K.gguf", + "ggml_metal_op_encode: error: unsupported op 'SOME_OP'", + "/tmp/ggml/src/ggml-metal.m:1234: fatal error", + *[f"{i} sd-server 0x000000010311{i:04x} ggml_print_backtrace + {i}" for i in range(24)], + ] + + tail = srv._diagnostic_tail(lines) + + assert "unsupported op 'SOME_OP'" in tail + assert "fatal error" in tail + # Still ends with recent context, so a failure with no marked line is not left empty. + assert "ggml_print_backtrace" in tail + + +def test_diagnostic_tail_falls_back_to_the_last_lines(): + lines = [f"step {i}" for i in range(50)] + tail = srv._diagnostic_tail(lines) + assert "step 49" in tail + assert "step 0" not in tail + + +def test_diagnostic_tail_is_bounded(): + lines = ["error: " + "x" * 500 for _ in range(20)] + assert len(srv._diagnostic_tail(lines)) <= 1500