Name the class of a failed generation instead of a bare "Image generation failed."

Found on a macOS runner: the native renderer aborts inside its own text encoder there, and the
page showed only "Image generation failed." with the sd-server backtrace left in the server log,
so nothing about the failure reached the user. The failure is now classified into fixed text, out
of memory and native-process death, so the message says what happened and what to try. None of the
engine's own output is echoed, since a native tail carries local paths and argv; that stays in the
log, and an unrecognised failure keeps the original literal.
This commit is contained in:
Daniel Han 2026-07-27 03:45:40 +00:00
commit a47cc6299d
2 changed files with 68 additions and 7 deletions

View file

@ -16197,6 +16197,37 @@ async def load_diffusion_model(
_diffusion_persist_active = 0
_GENERATE_FAILURE_FALLBACK = "Image generation failed."
# Failure classes worth naming in the UI, as FIXED text. The engine's own message can embed local
# paths and argv (a native tail, a Metal abort backtrace), so none of it is echoed: only the class
# is reported, and the full text stays in the server log.
_GENERATE_FAILURE_CLASSES: tuple[tuple[tuple[str, ...], str], ...] = (
(
("out of memory", "outofmemory", "oom"),
"The device ran out of memory. Try a smaller size, fewer steps, or a smaller batch.",
),
(
("sd-server connection lost", "sd-cli exited", "process exited", "ggml_abort", "signal"),
"The native image renderer stopped unexpectedly. Switch the engine to diffusers, or see "
"the server log for its output.",
),
)
def _generate_failure_detail(message: str) -> str:
"""A user-facing reason for a failed generation, built only from fixed text.
The bare literal left a real failure undiagnosable from the UI: on a Metal host the native
renderer aborts inside its own text encoder, and the page showed "Image generation failed."
with nothing to act on. Naming the CLASS of failure keeps the message useful without echoing
the engine's text, which can carry local paths and argv."""
text = str(message or "").lower()
for needles, detail in _GENERATE_FAILURE_CLASSES:
if any(n in text for n in needles):
return f"{_GENERATE_FAILURE_FALLBACK} {detail}"
return _GENERATE_FAILURE_FALLBACK
@studio_router.post("/images/generate", response_model = DiffusionGenerateResponse)
async def generate_diffusion_image(
request: DiffusionGenerateRequest, current_subject: str = Depends(get_current_subject)
@ -16254,7 +16285,7 @@ async def generate_diffusion_image(
if msg in (DIFFUSION_NOT_LOADED_MSG, DIFFUSION_CANCELLED_MSG):
raise HTTPException(status_code = 409, detail = msg)
logger.error("diffusion.generate_failed: %s", exc, exc_info = True)
raise HTTPException(status_code = 500, detail = "Image generation failed.")
raise HTTPException(status_code = 500, detail = _generate_failure_detail(msg))
except Exception as exc:
logger.error("diffusion.generate_failed: %s", exc, exc_info = True)
raise HTTPException(status_code = 500, detail = "Image generation failed.")

View file

@ -515,18 +515,47 @@ def test_generate_without_load_returns_409(client):
def test_generate_pipeline_error_returns_sanitized_500(client, monkeypatch):
# A loaded model that fails mid-pipeline (CUDA OOM, a RuntimeError) is a server failure: 500 with
# a generic message, not a 409 echoing the raw exception.
# a message built from FIXED text, not a 409 and not the raw exception. The class of failure is
# named so the page can suggest something; none of the engine's own text is echoed, since it can
# carry local paths and argv.
backend = diffusion_module.get_diffusion_backend()
backend.loaded = True
def _oom(**kwargs):
raise RuntimeError("CUDA out of memory. Tried to allocate 20.00 GiB (24.00 GiB total)")
raise RuntimeError(
"CUDA out of memory. Tried to allocate 20.00 GiB at /home/u/models/x.safetensors"
)
monkeypatch.setattr(backend, "generate", _oom)
resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
assert resp.status_code == 500
assert resp.json()["detail"] == "Image generation failed."
assert "CUDA" not in resp.json()["detail"]
detail = resp.json()["detail"]
assert detail.startswith("Image generation failed.")
assert "ran out of memory" in detail
for leak in ("CUDA", "20.00 GiB", "/home/u", "safetensors"):
assert leak not in detail
def test_generate_native_process_death_names_the_engine_not_its_output(client, monkeypatch):
# What a Metal host hits: the native renderer aborts inside its own text encoder. The page used
# to show only "Image generation failed."; it now says which component died, while the abort
# backtrace (full of local paths) stays in the server log.
backend = diffusion_module.get_diffusion_backend()
backend.loaded = True
def _abort(**kwargs):
raise RuntimeError(
"sd-server connection lost during img_gen poll (process exited, code -6)\n"
"Last output:\n0 sd-server ggml_abort + 156 at /Users/me/.cache/sd-cli"
)
monkeypatch.setattr(backend, "generate", _abort)
resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
assert resp.status_code == 500
detail = resp.json()["detail"]
assert "native image renderer stopped" in detail
for leak in ("ggml_abort", "/Users/me", "img_gen", "code -6"):
assert leak not in detail
def test_generate_execution_error_with_cancelled_substring_is_sanitized_500(client, monkeypatch):
@ -541,8 +570,9 @@ def test_generate_execution_error_with_cancelled_substring_is_sanitized_500(clie
monkeypatch.setattr(backend, "generate", _fail)
resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
assert resp.status_code == 500
assert resp.json()["detail"] == "Image generation failed."
assert "cancelled" not in resp.json()["detail"] and "models" not in resp.json()["detail"]
detail = resp.json()["detail"]
assert detail.startswith("Image generation failed.")
assert "cancelled" not in detail and "models" not in detail and "/home/u" not in detail
def test_generate_user_cancellation_returns_409(client, monkeypatch):