Studio: harden video/diffusion cache, attention, and CFG-parallel fault paths

- diffusion_attention: arch-gate FlashAttention 2 to Ampere (SM80)+ in both the
  primary selector and the heterogeneous-replica guard (it crashed on pre-Ampere).
- diffusion_cfg_parallel: convert boolean attn masks to additive bias before the direct
  cuDNN op so partial masks match F.scaled_dot_product_attention; make proxy disable_cache
  transactional (clean both branches, mark broken, surface a reload-required error).
- diffusion_cache: fail closed when a magcache step-count resize or below-threshold
  disable cannot remove the old cache; surface a failed enable+cleanup instead of a false
  uncached None.
- video: roll back earlier experts when a later expert raises in the all-or-none step-cache
  loop; fail the load when the primary-only cache cannot be re-engaged through the
  CFG-parallel proxy; validate transformer_cache_quality and cfg_parallel before the worker.
- scripts: place the fp8 ablation pipeline on CUDA; fail closed on a failed magcache resize
  in the speedmem bench; label OOM distinctly in the SDPA mask probe.
- tests: regressions for the FA2 arch gate, transactional proxy disable, all-or-none
  exception rollback, magcache fail-closed transitions, and enable+cleanup failure.
This commit is contained in:
Daniel Han 2026-07-13 09:46:17 +00:00
commit 133f6fecf7
13 changed files with 240 additions and 34 deletions

View file

@ -372,6 +372,49 @@ def test_toggle_reengages_after_a_disable(monkeypatch):
assert mode == TC_FBCACHE and t.enables == 2
def test_toggle_magcache_step_change_hard_errors_when_disable_fails(monkeypatch):
# A failed removal during a magcache step-count change used to fall through and report
# "magcache" while the OLD #sN curve stayed armed (wrong ratio schedule). Fail closed.
import core.inference.diffusion_cache as dc_mod
_stub_diffusers(monkeypatch)
t = _ToggleTransformer()
t._unsloth_step_cache = "magcache@0.06#s50"
monkeypatch.setattr(dc_mod, "_disengage_step_cache", lambda *a, **k: False)
with pytest.raises(RuntimeError, match = "reload the video model"):
maybe_toggle_step_cache(
_pipe(t), steps = 30, mode = dc_mod.TC_MAGCACHE, family = "hunyuanvideo-1.5"
)
def test_toggle_below_bar_hard_errors_when_disable_fails(monkeypatch):
# Below the cache threshold we want uncached; a failed disable used to report the stale
# mode instead of surfacing that the (wrong-step) cache is still armed.
import core.inference.diffusion_cache as dc_mod
_stub_diffusers(monkeypatch)
t = _ToggleTransformer()
t._unsloth_step_cache = "magcache@0.06#s50"
monkeypatch.setattr(dc_mod, "_disengage_step_cache", lambda *a, **k: False)
with pytest.raises(RuntimeError, match = "reload the video model"):
maybe_toggle_step_cache(
_pipe(t),
steps = FBCACHE_MIN_STEPS - 1,
mode = dc_mod.TC_MAGCACHE,
family = "hunyuanvideo-1.5",
)
def test_apply_step_cache_enable_and_cleanup_failure_requires_reload(monkeypatch):
# enable_cache fails AFTER partially hooking and the cleanup disable_cache ALSO fails:
# the transformer may keep partial hooks, so surface it instead of a clean uncached None.
_stub_diffusers(monkeypatch)
t = _MixinTransformer(fail = True)
t.disable_cache = lambda: (_ for _ in ()).throw(RuntimeError("cleanup failed"))
with pytest.raises(RuntimeError, match = "partially cached and must be reloaded"):
apply_step_cache(_pipe(t), mode = "fbcache")
def test_toggle_noop_without_cache_support(monkeypatch):
_stub_diffusers(monkeypatch)
t = _NonCacheMixinTransformer()