From a47cc6299d1f1404a50b76d77e831d98c1cbc289 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 27 Jul 2026 03:45:40 +0000 Subject: [PATCH] Name the class of a failed generation instead of a bare "Image generation failed." Found on a macOS runner: the native renderer aborts inside its own text encoder there, and the page showed only "Image generation failed." with the sd-server backtrace left in the server log, so nothing about the failure reached the user. The failure is now classified into fixed text, out of memory and native-process death, so the message says what happened and what to try. None of the engine's own output is echoed, since a native tail carries local paths and argv; that stays in the log, and an unrecognised failure keeps the original literal. --- studio/backend/routes/inference.py | 33 ++++++++++++++- studio/backend/tests/test_diffusion_routes.py | 42 ++++++++++++++++--- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index b22d61fee7..fd2f5f3980 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -16197,6 +16197,37 @@ async def load_diffusion_model( _diffusion_persist_active = 0 +_GENERATE_FAILURE_FALLBACK = "Image generation failed." +# Failure classes worth naming in the UI, as FIXED text. The engine's own message can embed local +# paths and argv (a native tail, a Metal abort backtrace), so none of it is echoed: only the class +# is reported, and the full text stays in the server log. +_GENERATE_FAILURE_CLASSES: tuple[tuple[tuple[str, ...], str], ...] = ( + ( + ("out of memory", "outofmemory", "oom"), + "The device ran out of memory. Try a smaller size, fewer steps, or a smaller batch.", + ), + ( + ("sd-server connection lost", "sd-cli exited", "process exited", "ggml_abort", "signal"), + "The native image renderer stopped unexpectedly. Switch the engine to diffusers, or see " + "the server log for its output.", + ), +) + + +def _generate_failure_detail(message: str) -> str: + """A user-facing reason for a failed generation, built only from fixed text. + + The bare literal left a real failure undiagnosable from the UI: on a Metal host the native + renderer aborts inside its own text encoder, and the page showed "Image generation failed." + with nothing to act on. Naming the CLASS of failure keeps the message useful without echoing + the engine's text, which can carry local paths and argv.""" + text = str(message or "").lower() + for needles, detail in _GENERATE_FAILURE_CLASSES: + if any(n in text for n in needles): + return f"{_GENERATE_FAILURE_FALLBACK} {detail}" + return _GENERATE_FAILURE_FALLBACK + + @studio_router.post("/images/generate", response_model = DiffusionGenerateResponse) async def generate_diffusion_image( request: DiffusionGenerateRequest, current_subject: str = Depends(get_current_subject) @@ -16254,7 +16285,7 @@ async def generate_diffusion_image( if msg in (DIFFUSION_NOT_LOADED_MSG, DIFFUSION_CANCELLED_MSG): raise HTTPException(status_code = 409, detail = msg) logger.error("diffusion.generate_failed: %s", exc, exc_info = True) - raise HTTPException(status_code = 500, detail = "Image generation failed.") + raise HTTPException(status_code = 500, detail = _generate_failure_detail(msg)) except Exception as exc: logger.error("diffusion.generate_failed: %s", exc, exc_info = True) raise HTTPException(status_code = 500, detail = "Image generation failed.") diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py index 360537c625..b9d47c0967 100644 --- a/studio/backend/tests/test_diffusion_routes.py +++ b/studio/backend/tests/test_diffusion_routes.py @@ -515,18 +515,47 @@ def test_generate_without_load_returns_409(client): def test_generate_pipeline_error_returns_sanitized_500(client, monkeypatch): # A loaded model that fails mid-pipeline (CUDA OOM, a RuntimeError) is a server failure: 500 with - # a generic message, not a 409 echoing the raw exception. + # a message built from FIXED text, not a 409 and not the raw exception. The class of failure is + # named so the page can suggest something; none of the engine's own text is echoed, since it can + # carry local paths and argv. backend = diffusion_module.get_diffusion_backend() backend.loaded = True def _oom(**kwargs): - raise RuntimeError("CUDA out of memory. Tried to allocate 20.00 GiB (24.00 GiB total)") + raise RuntimeError( + "CUDA out of memory. Tried to allocate 20.00 GiB at /home/u/models/x.safetensors" + ) monkeypatch.setattr(backend, "generate", _oom) resp = client.post("/api/inference/images/generate", json = {"prompt": "p"}) assert resp.status_code == 500 - assert resp.json()["detail"] == "Image generation failed." - assert "CUDA" not in resp.json()["detail"] + detail = resp.json()["detail"] + assert detail.startswith("Image generation failed.") + assert "ran out of memory" in detail + for leak in ("CUDA", "20.00 GiB", "/home/u", "safetensors"): + assert leak not in detail + + +def test_generate_native_process_death_names_the_engine_not_its_output(client, monkeypatch): + # What a Metal host hits: the native renderer aborts inside its own text encoder. The page used + # to show only "Image generation failed."; it now says which component died, while the abort + # backtrace (full of local paths) stays in the server log. + backend = diffusion_module.get_diffusion_backend() + backend.loaded = True + + def _abort(**kwargs): + raise RuntimeError( + "sd-server connection lost during img_gen poll (process exited, code -6)\n" + "Last output:\n0 sd-server ggml_abort + 156 at /Users/me/.cache/sd-cli" + ) + + monkeypatch.setattr(backend, "generate", _abort) + resp = client.post("/api/inference/images/generate", json = {"prompt": "p"}) + assert resp.status_code == 500 + detail = resp.json()["detail"] + assert "native image renderer stopped" in detail + for leak in ("ggml_abort", "/Users/me", "img_gen", "code -6"): + assert leak not in detail def test_generate_execution_error_with_cancelled_substring_is_sanitized_500(client, monkeypatch): @@ -541,8 +570,9 @@ def test_generate_execution_error_with_cancelled_substring_is_sanitized_500(clie monkeypatch.setattr(backend, "generate", _fail) resp = client.post("/api/inference/images/generate", json = {"prompt": "p"}) assert resp.status_code == 500 - assert resp.json()["detail"] == "Image generation failed." - assert "cancelled" not in resp.json()["detail"] and "models" not in resp.json()["detail"] + detail = resp.json()["detail"] + assert detail.startswith("Image generation failed.") + assert "cancelled" not in detail and "models" not in detail and "/home/u" not in detail def test_generate_user_cancellation_returns_409(client, monkeypatch):