Name the class of a failed generation instead of a bare "Image generation failed."
Found on a macOS runner: the native renderer aborts inside its own text encoder there, and the page showed only "Image generation failed." with the sd-server backtrace left in the server log, so nothing about the failure reached the user. The failure is now classified into fixed text, out of memory and native-process death, so the message says what happened and what to try. None of the engine's own output is echoed, since a native tail carries local paths and argv; that stays in the log, and an unrecognised failure keeps the original literal.
This commit is contained in:
parent
bfbcc53e39
commit
a47cc6299d
2 changed files with 68 additions and 7 deletions
|
|
@ -16197,6 +16197,37 @@ async def load_diffusion_model(
|
|||
_diffusion_persist_active = 0
|
||||
|
||||
|
||||
_GENERATE_FAILURE_FALLBACK = "Image generation failed."
|
||||
# Failure classes worth naming in the UI, as FIXED text. The engine's own message can embed local
|
||||
# paths and argv (a native tail, a Metal abort backtrace), so none of it is echoed: only the class
|
||||
# is reported, and the full text stays in the server log.
|
||||
_GENERATE_FAILURE_CLASSES: tuple[tuple[tuple[str, ...], str], ...] = (
|
||||
(
|
||||
("out of memory", "outofmemory", "oom"),
|
||||
"The device ran out of memory. Try a smaller size, fewer steps, or a smaller batch.",
|
||||
),
|
||||
(
|
||||
("sd-server connection lost", "sd-cli exited", "process exited", "ggml_abort", "signal"),
|
||||
"The native image renderer stopped unexpectedly. Switch the engine to diffusers, or see "
|
||||
"the server log for its output.",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _generate_failure_detail(message: str) -> str:
|
||||
"""A user-facing reason for a failed generation, built only from fixed text.
|
||||
|
||||
The bare literal left a real failure undiagnosable from the UI: on a Metal host the native
|
||||
renderer aborts inside its own text encoder, and the page showed "Image generation failed."
|
||||
with nothing to act on. Naming the CLASS of failure keeps the message useful without echoing
|
||||
the engine's text, which can carry local paths and argv."""
|
||||
text = str(message or "").lower()
|
||||
for needles, detail in _GENERATE_FAILURE_CLASSES:
|
||||
if any(n in text for n in needles):
|
||||
return f"{_GENERATE_FAILURE_FALLBACK} {detail}"
|
||||
return _GENERATE_FAILURE_FALLBACK
|
||||
|
||||
|
||||
@studio_router.post("/images/generate", response_model = DiffusionGenerateResponse)
|
||||
async def generate_diffusion_image(
|
||||
request: DiffusionGenerateRequest, current_subject: str = Depends(get_current_subject)
|
||||
|
|
@ -16254,7 +16285,7 @@ async def generate_diffusion_image(
|
|||
if msg in (DIFFUSION_NOT_LOADED_MSG, DIFFUSION_CANCELLED_MSG):
|
||||
raise HTTPException(status_code = 409, detail = msg)
|
||||
logger.error("diffusion.generate_failed: %s", exc, exc_info = True)
|
||||
raise HTTPException(status_code = 500, detail = "Image generation failed.")
|
||||
raise HTTPException(status_code = 500, detail = _generate_failure_detail(msg))
|
||||
except Exception as exc:
|
||||
logger.error("diffusion.generate_failed: %s", exc, exc_info = True)
|
||||
raise HTTPException(status_code = 500, detail = "Image generation failed.")
|
||||
|
|
|
|||
|
|
@ -515,18 +515,47 @@ def test_generate_without_load_returns_409(client):
|
|||
|
||||
def test_generate_pipeline_error_returns_sanitized_500(client, monkeypatch):
|
||||
# A loaded model that fails mid-pipeline (CUDA OOM, a RuntimeError) is a server failure: 500 with
|
||||
# a generic message, not a 409 echoing the raw exception.
|
||||
# a message built from FIXED text, not a 409 and not the raw exception. The class of failure is
|
||||
# named so the page can suggest something; none of the engine's own text is echoed, since it can
|
||||
# carry local paths and argv.
|
||||
backend = diffusion_module.get_diffusion_backend()
|
||||
backend.loaded = True
|
||||
|
||||
def _oom(**kwargs):
|
||||
raise RuntimeError("CUDA out of memory. Tried to allocate 20.00 GiB (24.00 GiB total)")
|
||||
raise RuntimeError(
|
||||
"CUDA out of memory. Tried to allocate 20.00 GiB at /home/u/models/x.safetensors"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(backend, "generate", _oom)
|
||||
resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
|
||||
assert resp.status_code == 500
|
||||
assert resp.json()["detail"] == "Image generation failed."
|
||||
assert "CUDA" not in resp.json()["detail"]
|
||||
detail = resp.json()["detail"]
|
||||
assert detail.startswith("Image generation failed.")
|
||||
assert "ran out of memory" in detail
|
||||
for leak in ("CUDA", "20.00 GiB", "/home/u", "safetensors"):
|
||||
assert leak not in detail
|
||||
|
||||
|
||||
def test_generate_native_process_death_names_the_engine_not_its_output(client, monkeypatch):
|
||||
# What a Metal host hits: the native renderer aborts inside its own text encoder. The page used
|
||||
# to show only "Image generation failed."; it now says which component died, while the abort
|
||||
# backtrace (full of local paths) stays in the server log.
|
||||
backend = diffusion_module.get_diffusion_backend()
|
||||
backend.loaded = True
|
||||
|
||||
def _abort(**kwargs):
|
||||
raise RuntimeError(
|
||||
"sd-server connection lost during img_gen poll (process exited, code -6)\n"
|
||||
"Last output:\n0 sd-server ggml_abort + 156 at /Users/me/.cache/sd-cli"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(backend, "generate", _abort)
|
||||
resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
|
||||
assert resp.status_code == 500
|
||||
detail = resp.json()["detail"]
|
||||
assert "native image renderer stopped" in detail
|
||||
for leak in ("ggml_abort", "/Users/me", "img_gen", "code -6"):
|
||||
assert leak not in detail
|
||||
|
||||
|
||||
def test_generate_execution_error_with_cancelled_substring_is_sanitized_500(client, monkeypatch):
|
||||
|
|
@ -541,8 +570,9 @@ def test_generate_execution_error_with_cancelled_substring_is_sanitized_500(clie
|
|||
monkeypatch.setattr(backend, "generate", _fail)
|
||||
resp = client.post("/api/inference/images/generate", json = {"prompt": "p"})
|
||||
assert resp.status_code == 500
|
||||
assert resp.json()["detail"] == "Image generation failed."
|
||||
assert "cancelled" not in resp.json()["detail"] and "models" not in resp.json()["detail"]
|
||||
detail = resp.json()["detail"]
|
||||
assert detail.startswith("Image generation failed.")
|
||||
assert "cancelled" not in detail and "models" not in detail and "/home/u" not in detail
|
||||
|
||||
|
||||
def test_generate_user_cancellation_returns_409(client, monkeypatch):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue