Studio: harden video/diffusion cache, attention, and CFG-parallel fault paths

- diffusion_attention: arch-gate FlashAttention 2 to Ampere (SM80)+ in both the
  primary selector and the heterogeneous-replica guard (it crashed on pre-Ampere).
- diffusion_cfg_parallel: convert boolean attn masks to additive bias before the direct
  cuDNN op so partial masks match F.scaled_dot_product_attention; make proxy disable_cache
  transactional (clean both branches, mark broken, surface a reload-required error).
- diffusion_cache: fail closed when a magcache step-count resize or below-threshold
  disable cannot remove the old cache; surface a failed enable+cleanup instead of a false
  uncached None.
- video: roll back earlier experts when a later expert raises in the all-or-none step-cache
  loop; fail the load when the primary-only cache cannot be re-engaged through the
  CFG-parallel proxy; validate transformer_cache_quality and cfg_parallel before the worker.
- scripts: place the fp8 ablation pipeline on CUDA; fail closed on a failed magcache resize
  in the speedmem bench; label OOM distinctly in the SDPA mask probe.
- tests: regressions for the FA2 arch gate, transactional proxy disable, all-or-none
  exception rollback, magcache fail-closed transitions, and enable+cleanup failure.
This commit is contained in:
Daniel Han 2026-07-13 09:46:17 +00:00
commit 133f6fecf7
13 changed files with 240 additions and 34 deletions

View file

@ -675,10 +675,20 @@ def apply_step_cache(
# reported-uncached model isn't half-cached. Restore armed compiled inners FIRST
# (remove_hook splices original_forward back into module.forward).
_restore_hooked_block_inners(transformer)
try:
transformer.disable_cache()
except Exception: # noqa: BLE001
pass
disable_cache = getattr(transformer, "disable_cache", None)
if callable(disable_cache):
try:
disable_cache()
transformer._unsloth_step_cache = None
except Exception as rollback_exc: # noqa: BLE001
# Both the enable AND its cleanup failed: the transformer may keep partial hooks
# while we'd otherwise report a clean uncached None. Surface it so the caller
# reloads instead of generating on a half-cached model.
raise RuntimeError(
"step-cache enable failed and rollback also failed; the transformer may be "
"partially cached and must be reloaded "
f"(enable error: {exc}; rollback error: {rollback_exc})"
) from rollback_exc
_warn(logger, mode, exc)
return None
@ -770,10 +780,17 @@ def maybe_toggle_step_cache(
and mode == TC_MAGCACHE
# endswith, not substring: "#s5" would match inside "#s50".
and not str(engaged).endswith(f"#s{int(steps)}")
and _disengage_step_cache(
transformer, reason = f"magcache re-interpolating for {steps} steps", logger = logger
)
):
# A failed removal used to short-circuit and fall through to `return mode`, reporting
# "magcache" while the OLD #sN curve stayed armed (wrong ratio schedule, silently
# degraded output). Fail closed so the caller reloads instead.
if not _disengage_step_cache(
transformer, reason = f"magcache re-interpolating for {steps} steps", logger = logger
):
raise RuntimeError(
"could not disable the existing MagCache before resizing it for "
f"{steps} steps; reload the video model before generating"
)
engaged = None
if want and not engaged:
return apply_step_cache(
@ -788,13 +805,18 @@ def maybe_toggle_step_cache(
logger = logger,
)
if not want and engaged:
if _disengage_step_cache(
# Below the cache threshold we want uncached; a failed disable leaves the (possibly
# wrong-step) cache armed, so surface it rather than reporting the stale mode.
if not _disengage_step_cache(
transformer,
reason = f"auto: {steps} steps < {FBCACHE_MIN_STEPS}",
logger = logger,
):
return None
return mode
raise RuntimeError(
"could not disable the existing step cache for a short generation; "
"reload the video model before generating"
)
return None
return mode if engaged else None