Stream every DiT through group offload, not just the primary transformer

A dual-DiT pipeline (Ideogram 4's unconditional tower) placed its second
denoiser resident under the group tier, which defeats the tier since the
pair rarely fits where one alone did not. Stream transformer_2 and
unconditional_transformer alongside the transformer and keep only the
smaller companions resident.
This commit is contained in:
Daniel Han 2026-07-05 05:30:58 +00:00
commit 7a05d8655b

View file

@ -502,6 +502,16 @@ def _apply_group_offload(pipe: Any, device: str, logger: Any) -> bool:
import torch
from diffusers.hooks import apply_group_offloading
# A dual-DiT pipeline (e.g. Ideogram 4's unconditional tower) carries a second
# denoiser as large as the first; leaving it resident would defeat this tier
# (the pair rarely fits where one alone did not). Stream every DiT and keep
# only the genuinely smaller companions resident.
streamed: dict[str, Any] = {"transformer": transformer}
for extra in ("transformer_2", "unconditional_transformer"):
module = getattr(pipe, extra, None)
if isinstance(module, torch.nn.Module):
streamed[extra] = module
onload = torch.device(device)
use_stream = onload.type == "cuda" # overlap H2D copies with compute on CUDA
gkwargs: dict[str, Any] = {
@ -531,11 +541,12 @@ def _apply_group_offload(pipe: Any, device: str, logger: Any) -> bool:
# load-time crash. The streamed transformer manages its own placement via the
# offloading hooks applied next.
for name, comp in getattr(pipe, "components", {}).items():
if name == "transformer":
if name in streamed:
continue
if isinstance(comp, torch.nn.Module):
comp.to(onload)
apply_group_offloading(transformer, **gkwargs)
for module in streamed.values():
apply_group_offloading(module, **gkwargs)
return True
except Exception as exc: # noqa: BLE001 — fall back to whole-module offload
if logger is not None: