Size-gate VAE auto-quant: quantize large (video) VAEs only, skip tiny image VAEs

A B200 speed/memory sweep (new scripts/quant_speedmem_bench.py) shows the VAE quant
win is a video story. Image AutoencoderKLs are ~0.15-0.26 GB, so fp8 saves ~0.1 GB
and only slows their tiny decode (+6-16%); the video Conv3d VAEs are ~2.5 GB and
halve to ~1.2 GB at ~2% decode cost. So VAE auto now only engages above a ~1 GB size
floor: small image VAEs stay dense (faster decode, no quant quality risk), video VAEs
still quantize. An explicit fp8 / fp8_dynamic request skips the gate (opted in).

The same sweep confirmed the text-encoder default is already right: fp8_dynamic is
E2E-neutral (denoise per-step unchanged; +2% one-time encode) and, by hidden-state
cosine vs bf16, marginally more accurate than layerwise fp8 -- so that default is left
as is. Tests cover the gate (small skipped, large quantized, explicit bypasses).

studio/backend/tests/test_diffusion_vae_quant.py

@@ -393,10 +393,49 @@ def test_quantize_vae_auto_resolves_and_applies(monkeypatch):
     monkeypatch.setattr(vq, "_cast_vae_fp8", lambda v, t: calls.append(v))
     vae = object()
     pipe = types.SimpleNamespace(vae = vae)
+    # large VAE -> clears the size gate
+    monkeypatch.setattr(vq, "_vae_param_bytes", lambda v: 3_000_000_000)
     assert quantize_vae(pipe, _target(), mode = "auto", family = "flux.1") == VAE_QUANT_FP8
     assert calls == [vae]
 
 
+def test_quantize_vae_auto_size_gate_skips_small(monkeypatch):
+    # auto leaves a small (image) VAE dense: halving ~0.2GB saves ~nothing and only slows decode.
+    _stub_torch(monkeypatch, cc = (10, 0))
+    _stub_capability(monkeypatch, (10, 0))
+    _allow_vae(monkeypatch, {VAE_QUANT_FP8_DYNAMIC, VAE_QUANT_FP8})
+    monkeypatch.setattr(vq, "_vae_param_bytes", lambda v: 200_000_000)  # ~0.2 GB image VAE
+    monkeypatch.setattr(vq, "_cast_vae_fp8", lambda v, t: pytest.fail("small VAE must stay dense"))
+    monkeypatch.setattr(vq, "_cast_vae_fp8_dynamic", lambda v, t: pytest.fail("small VAE must stay dense"))
+    pipe = types.SimpleNamespace(vae = object())
+    assert quantize_vae(pipe, _target(), mode = "auto", family = "flux.1") is None
+
+
+def test_quantize_vae_auto_size_gate_allows_large(monkeypatch):
+    # auto quantises a large (video Conv3d) VAE: halving ~2.5GB saves ~1.2GB at ~2% decode.
+    _stub_torch(monkeypatch, cc = (10, 0))
+    _stub_capability(monkeypatch, (10, 0))
+    _allow_vae(monkeypatch, {VAE_QUANT_FP8_DYNAMIC, VAE_QUANT_FP8})
+    monkeypatch.setattr(vq, "_vae_param_bytes", lambda v: 2_500_000_000)  # ~2.5 GB video VAE
+    calls: list = []
+    monkeypatch.setattr(vq, "_cast_vae_fp8", lambda v, t: calls.append(v))
+    pipe = types.SimpleNamespace(vae = object())
+    assert quantize_vae(pipe, _target(), mode = "auto", family = "hunyuanvideo-1.5") == VAE_QUANT_FP8
+    assert len(calls) == 1
+
+
+def test_quantize_vae_explicit_bypasses_size_gate(monkeypatch):
+    # An explicit request quantises even a small VAE -- the user opted in directly.
+    _stub_torch(monkeypatch, cc = (10, 0))
+    _allow_vae(monkeypatch, {VAE_QUANT_FP8})
+    monkeypatch.setattr(vq, "_vae_param_bytes", lambda v: 200_000_000)  # small, but explicit
+    calls: list = []
+    monkeypatch.setattr(vq, "_cast_vae_fp8", lambda v, t: calls.append(v))
+    pipe = types.SimpleNamespace(vae = object())
+    assert quantize_vae(pipe, _target(), mode = "fp8", family = "flux.1") == VAE_QUANT_FP8
+    assert len(calls) == 1
+
+
 def test_quantize_vae_explicit_fp8_dynamic_probe_gates(monkeypatch):
     # An explicit fp8_dynamic request runs the conv smoke probe: a build whose torchao lacks a
     # working fp8 conv path (probe False) stays dense; a passing probe applies the caster.