perf(image): compile numeric parity, cache-hook compile arming, FBCache toggle crash fix, TE fp8 zero-row guard
Applies the video round-2 accuracy findings to the image diffusion stack and fixes two real image-path bugs found while measuring. All numbers B200, production settings (family default steps/guidance, 1024px, seed 42, 4 fixed prompts), LPIPS (AlexNet) via the new scripts/image_speedmem_bench.py, which drives the production lever functions in the loader's own order. - inductor precision parity: emulate_precision_casts=True on the regional-compile path (fused pointwise kernels keep fp32 intermediates where eager rounds to bf16 between ops). Pairwise LPIPS of the compiled tier vs the same-stack eager tier: Qwen-Image 0.019 to 0.006 at identical speed (72.4 vs 72.5 ms/step), FLUX.1-dev 0.046 to 0.029 at +2% step time (69.8 vs 68.3, reproduced), FLUX.2-klein-4B 0.018 to 0.017 at identical speed. Snapshot/restored with the other process-wide backend flags so an off load never inherits it. - cache x compile composition: re-point each cache hook's fn_ref.original_forward at a torch.compile'd wrapper of the same bound method (armed only where the speed layer compiled the block; restored before every disable_cache and before the partial-hook cleanup). Qwen-Image FBCache computed steps 91.8 to 71.2 ms (back at the uncached compiled rate), 1.21x end to end (7.36 to 6.06 s per 4 images); FLUX.1-dev already traced through its FBCache hook and is measured neutral (same-process armed vs unarmed latents bit-identical). Skip counts within noise (13 vs 11 of 76; pairwise LPIPS 0.005). - FBCache mid-session toggle crash: diffusers 0.39 caches the HookRegistry child list on first cache_context use, so an uncached generation followed by a 20+-step generation (the auto toggle path) enabled hooks the context never reached and crashed with "No context is set" (reproduced live on FLUX.1-dev). Invalidate the stale child cache after every enable_cache. - TE fp8_dynamic zero-row guard: torchao per-row fp8 derives a per-output-channel scale from the row amax, so an all-zero weight row is 0/0 = NaN. SDXL's text_encoder_2 (OpenCLIP bigG) ships exactly such a row, and every explicit fp8_dynamic SDXL render came out black; keep zero-row Linears dense (LPIPS 0.976 black to 0.096 working). Other families' encoders have no such rows and are byte-identical. - No AUTO TE quant exists on the image branch (text_encoder_quant defaults dense, explicit-only), so the video round's auto-dense retune has no image analogue; the explicit lever's cost is now measured (TE fp8_dynamic alone, LPIPS vs bit-exact: Qwen-Image 0.038, FLUX.1-dev 0.084, SDXL 0.096; no speed win, VRAM -6.5 GB on Qwen-Image) for the docs. Tests: 96 passing across the cache/speed/precision suites (11 new arming, 2 child-registry, 2 zero-row, 4 inductor-flag); ruff clean.
This commit is contained in:
parent
ec90b8658d
commit
de2f22df2b
7 changed files with 997 additions and 2 deletions
|
|
@ -77,6 +77,9 @@ def snapshot_backend_flags() -> Optional[dict]:
|
|||
state["cudnn_tf32"] = bool(cudnn.allow_tf32)
|
||||
if hasattr(cudnn, "benchmark"):
|
||||
state["cudnn_benchmark"] = bool(cudnn.benchmark)
|
||||
inductor_cfg = _inductor_config()
|
||||
if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):
|
||||
state["inductor_emulate_precision_casts"] = bool(inductor_cfg.emulate_precision_casts)
|
||||
return state
|
||||
|
||||
|
||||
|
|
@ -103,6 +106,19 @@ def restore_backend_flags(state: Optional[dict]) -> None:
|
|||
cudnn = getattr(torch.backends, "cudnn", None)
|
||||
_set(cudnn, "allow_tf32", "cudnn_tf32")
|
||||
_set(cudnn, "benchmark", "cudnn_benchmark")
|
||||
_set(_inductor_config(), "emulate_precision_casts", "inductor_emulate_precision_casts")
|
||||
|
||||
|
||||
def _inductor_config() -> Any:
|
||||
"""``torch._inductor.config`` or None. Resolved as attributes off the imported torch
|
||||
module (real torch exposes ``_inductor`` directly after ``import torch``) rather
|
||||
than a submodule import, so a stubbed/partial torch (tests, exotic builds) cleanly
|
||||
reports None instead of picking a stale real module out of ``sys.modules``."""
|
||||
try:
|
||||
import torch
|
||||
return getattr(getattr(torch, "_inductor", None), "config", None)
|
||||
except Exception: # noqa: BLE001 — no inductor -> nothing to snapshot/set
|
||||
return None
|
||||
|
||||
|
||||
def normalize_speed_mode(value: Optional[str]) -> str:
|
||||
|
|
@ -342,6 +358,19 @@ def _compile_repeated_blocks(
|
|||
for _limit_attr in ("recompile_limit", "cache_size_limit"): # name varies by torch ver
|
||||
if hasattr(dynamo_cfg, _limit_attr):
|
||||
setattr(dynamo_cfg, _limit_attr, max(getattr(dynamo_cfg, _limit_attr) or 0, 64))
|
||||
# Match eager's intermediate rounding inside inductor's fused pointwise kernels:
|
||||
# by default they keep chains in fp32 where eager materialises bf16 between ops,
|
||||
# a per-forward rounding delta that a multi-step denoise amplifies chaotically.
|
||||
# Measured (B200, scripts/image_speedmem_bench.py, pairwise LPIPS of the
|
||||
# compiled tier vs the same-stack eager tier): Qwen-Image 0.019 -> 0.006 at
|
||||
# identical speed, FLUX.1-dev 0.046 -> 0.029 at +2% step time, FLUX.2-klein
|
||||
# 0.018 -> 0.017 at identical speed; on the video DiT (HunyuanVideo-1.5-720p)
|
||||
# full-clip LPIPS vs bit-exact drops 0.221 -> 0.052 at zero cost. Process-
|
||||
# global, so snapshot_backend_flags carries it and unload restores the prior
|
||||
# value.
|
||||
inductor_cfg = _inductor_config()
|
||||
if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):
|
||||
inductor_cfg.emulate_precision_casts = True
|
||||
except Exception as exc: # noqa: BLE001 — optimisation only
|
||||
_warn(logger, "compile_repeated_blocks", exc)
|
||||
return False
|
||||
|
|
@ -354,6 +383,20 @@ def _compile_repeated_blocks(
|
|||
engaged = True
|
||||
except Exception as exc: # noqa: BLE001 — optimisation only
|
||||
_warn(logger, "compile_repeated_blocks", exc)
|
||||
continue
|
||||
# A step cache engaged BEFORE this compile (the production load order) has
|
||||
# already wrapped each block's forward in a @torch.compiler.disable'd hook, so
|
||||
# the compute branch would run eager on every non-skipped step and forfeit the
|
||||
# regional compile entirely. Re-point the hooks' inner forward at compiled
|
||||
# wrappers; no-op when no cache hooks are installed. The toggle path (cache
|
||||
# engaged after load) is armed by apply_step_cache instead. Lazy import:
|
||||
# diffusion_cache imports nothing from this module, but keep the dependency
|
||||
# one-directional at import time.
|
||||
try:
|
||||
from .diffusion_cache import _compile_hooked_block_inners
|
||||
_compile_hooked_block_inners(transformer, logger)
|
||||
except Exception as exc: # noqa: BLE001 — optimisation only
|
||||
_warn(logger, "cache-hook inner compile", exc)
|
||||
return engaged
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue