Studio: harden video/diffusion cache, attention, and CFG-parallel fault paths
- diffusion_attention: arch-gate FlashAttention 2 to Ampere (SM80)+ in both the primary selector and the heterogeneous-replica guard (it crashed on pre-Ampere). - diffusion_cfg_parallel: convert boolean attn masks to additive bias before the direct cuDNN op so partial masks match F.scaled_dot_product_attention; make proxy disable_cache transactional (clean both branches, mark broken, surface a reload-required error). - diffusion_cache: fail closed when a magcache step-count resize or below-threshold disable cannot remove the old cache; surface a failed enable+cleanup instead of a false uncached None. - video: roll back earlier experts when a later expert raises in the all-or-none step-cache loop; fail the load when the primary-only cache cannot be re-engaged through the CFG-parallel proxy; validate transformer_cache_quality and cfg_parallel before the worker. - scripts: place the fp8 ablation pipeline on CUDA; fail closed on a failed magcache resize in the speedmem bench; label OOM distinctly in the SDPA mask probe. - tests: regressions for the FA2 arch gate, transactional proxy disable, all-or-none exception rollback, magcache fail-closed transitions, and enable+cleanup failure.
This commit is contained in:
parent
9eadfc822d
commit
133f6fecf7
13 changed files with 240 additions and 34 deletions
|
|
@ -372,6 +372,49 @@ def test_toggle_reengages_after_a_disable(monkeypatch):
|
|||
assert mode == TC_FBCACHE and t.enables == 2
|
||||
|
||||
|
||||
def test_toggle_magcache_step_change_hard_errors_when_disable_fails(monkeypatch):
|
||||
# A failed removal during a magcache step-count change used to fall through and report
|
||||
# "magcache" while the OLD #sN curve stayed armed (wrong ratio schedule). Fail closed.
|
||||
import core.inference.diffusion_cache as dc_mod
|
||||
|
||||
_stub_diffusers(monkeypatch)
|
||||
t = _ToggleTransformer()
|
||||
t._unsloth_step_cache = "magcache@0.06#s50"
|
||||
monkeypatch.setattr(dc_mod, "_disengage_step_cache", lambda *a, **k: False)
|
||||
with pytest.raises(RuntimeError, match = "reload the video model"):
|
||||
maybe_toggle_step_cache(
|
||||
_pipe(t), steps = 30, mode = dc_mod.TC_MAGCACHE, family = "hunyuanvideo-1.5"
|
||||
)
|
||||
|
||||
|
||||
def test_toggle_below_bar_hard_errors_when_disable_fails(monkeypatch):
|
||||
# Below the cache threshold we want uncached; a failed disable used to report the stale
|
||||
# mode instead of surfacing that the (wrong-step) cache is still armed.
|
||||
import core.inference.diffusion_cache as dc_mod
|
||||
|
||||
_stub_diffusers(monkeypatch)
|
||||
t = _ToggleTransformer()
|
||||
t._unsloth_step_cache = "magcache@0.06#s50"
|
||||
monkeypatch.setattr(dc_mod, "_disengage_step_cache", lambda *a, **k: False)
|
||||
with pytest.raises(RuntimeError, match = "reload the video model"):
|
||||
maybe_toggle_step_cache(
|
||||
_pipe(t),
|
||||
steps = FBCACHE_MIN_STEPS - 1,
|
||||
mode = dc_mod.TC_MAGCACHE,
|
||||
family = "hunyuanvideo-1.5",
|
||||
)
|
||||
|
||||
|
||||
def test_apply_step_cache_enable_and_cleanup_failure_requires_reload(monkeypatch):
|
||||
# enable_cache fails AFTER partially hooking and the cleanup disable_cache ALSO fails:
|
||||
# the transformer may keep partial hooks, so surface it instead of a clean uncached None.
|
||||
_stub_diffusers(monkeypatch)
|
||||
t = _MixinTransformer(fail = True)
|
||||
t.disable_cache = lambda: (_ for _ in ()).throw(RuntimeError("cleanup failed"))
|
||||
with pytest.raises(RuntimeError, match = "partially cached and must be reloaded"):
|
||||
apply_step_cache(_pipe(t), mode = "fbcache")
|
||||
|
||||
|
||||
def test_toggle_noop_without_cache_support(monkeypatch):
|
||||
_stub_diffusers(monkeypatch)
|
||||
t = _NonCacheMixinTransformer()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue