Studio diffusion (Phase 10): attention-backend selection
Add a selectable attention kernel via the diffusers set_attention_backend dispatcher. Attention is memory-bandwidth bound, so a better kernel is an end-to-end win orthogonal to the linear-weight quantisation (it speeds the QK/PV matmuls torchao never touches) and composes with torch.compile. auto picks the best exact backend for the device: cuDNN fused attention (_native_cudnn) on NVIDIA when a speed profile is active, measured ~1.18x end-to-end on a B200 (Z-Image 1024px/8 steps) with LPIPS ~0.004 vs the default (below the compile/quant noise floor); native SDPA elsewhere and when speed=off (so off stays bit-identical). Explicit native/cudnn/flash/flash3/flash4/sage/ xformers/aiter are honored, and an unavailable kernel falls back to the default rather than failing the load. New core/inference/diffusion_attention.py (normalize + per-device select + apply, best-effort, lazy imports). Set on pipe.transformer BEFORE compile in load_pipeline; attention_backend threads through begin_load / load_pipeline / status like the other load knobs. New request field attention_backend + status field. Hermetic CPU tests for normalize / select policy / apply fallback, plus route threading + 422. Measured via scripts/perf_levers_probe.py.
This commit is contained in:
parent
520b80a082
commit
b923675549
7 changed files with 490 additions and 0 deletions
|
|
@ -51,6 +51,10 @@ from .diffusion_speed import (
|
|||
restore_backend_flags,
|
||||
snapshot_backend_flags,
|
||||
)
|
||||
from .diffusion_attention import (
|
||||
apply_attention_backend,
|
||||
select_attention_backend,
|
||||
)
|
||||
from .diffusion_precision import quantize_text_encoders
|
||||
from .diffusion_prequant import (
|
||||
load_prequantized_transformer,
|
||||
|
|
@ -94,6 +98,9 @@ class _LoadState:
|
|||
# Transformer quant actually engaged on the opt-in dense fast path: "int8" | "fp8"
|
||||
# | "nvfp4" | "mxfp8" | None. None means the default GGUF transformer was loaded.
|
||||
transformer_quant: Optional[str] = None
|
||||
# Attention backend engaged via the diffusers dispatcher (e.g. "_native_cudnn"), or
|
||||
# None for the default SDPA. Set before compile; orthogonal to the weight quant.
|
||||
attention_backend: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -283,6 +290,7 @@ class DiffusionBackend:
|
|||
transformer_quant: Optional[str] = None,
|
||||
transformer_quant_fast_accum: Optional[bool] = None,
|
||||
transformer_prequant_path: Optional[str] = None,
|
||||
attention_backend: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Validate, then run the (slow) load on a daemon thread. Returns at once."""
|
||||
fam = self.validate_load_request(
|
||||
|
|
@ -317,6 +325,7 @@ class DiffusionBackend:
|
|||
transformer_quant = transformer_quant,
|
||||
transformer_quant_fast_accum = transformer_quant_fast_accum,
|
||||
transformer_prequant_path = transformer_prequant_path,
|
||||
attention_backend = attention_backend,
|
||||
_load_token = token,
|
||||
),
|
||||
daemon = True,
|
||||
|
|
@ -441,6 +450,7 @@ class DiffusionBackend:
|
|||
transformer_quant: Optional[str] = None,
|
||||
transformer_quant_fast_accum: Optional[bool] = None,
|
||||
transformer_prequant_path: Optional[str] = None,
|
||||
attention_backend: Optional[str] = None,
|
||||
_load_token: Optional[int] = None,
|
||||
) -> dict[str, Any]:
|
||||
# Validate first (cheap, no torch/diffusers) so a direct call with a bad
|
||||
|
|
@ -549,6 +559,18 @@ class DiffusionBackend:
|
|||
# first so unload can restore them: TF32 / cudnn.benchmark are global,
|
||||
# and a later `off` load must not inherit this load's settings.
|
||||
backend_flags_before = snapshot_backend_flags()
|
||||
# Pick the attention kernel BEFORE compile (compile traces attention). auto
|
||||
# upgrades to cuDNN fused attention on NVIDIA when a speed profile is active
|
||||
# (~1.18x, near-lossless); an explicit backend is honored, falling back to
|
||||
# the diffusers default if its kernel is unavailable. Orthogonal to the
|
||||
# weight quant -- it speeds the QK/PV matmuls torchao does not touch.
|
||||
attention_engaged = apply_attention_backend(
|
||||
pipe,
|
||||
select_attention_backend(
|
||||
target, attention_backend, speed_active = effective_speed != SPEED_OFF
|
||||
),
|
||||
logger = logger,
|
||||
)
|
||||
speed_applied = apply_speed_optims(
|
||||
pipe,
|
||||
target,
|
||||
|
|
@ -592,6 +614,7 @@ class DiffusionBackend:
|
|||
backend_flags_before = backend_flags_before,
|
||||
text_encoder_quant = te_quant,
|
||||
transformer_quant = transformer_quant_engaged,
|
||||
attention_backend = attention_engaged,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
|
|
@ -893,6 +916,7 @@ class DiffusionBackend:
|
|||
"speed_optims": [],
|
||||
"text_encoder_quant": None,
|
||||
"transformer_quant": None,
|
||||
"attention_backend": None,
|
||||
}
|
||||
return {
|
||||
"loaded": True,
|
||||
|
|
@ -909,6 +933,7 @@ class DiffusionBackend:
|
|||
"speed_optims": list(state.speed_optims),
|
||||
"text_encoder_quant": state.text_encoder_quant,
|
||||
"transformer_quant": state.transformer_quant,
|
||||
"attention_backend": state.attention_backend,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue