diff --git a/scripts/hunyuan_int8_profile.py b/scripts/hunyuan_int8_profile.py new file mode 100644 index 0000000000..b840007d79 --- /dev/null +++ b/scripts/hunyuan_int8_profile.py @@ -0,0 +1,155 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Why is HunyuanVideo-1.5 int8 DiT slower than dense+compile? Per-DiT-forward timing + dynamo +recompile/graph-break counting for dense vs int8, both under the loader's regional block compile. + +Signal: +- steady-state per-forward time uniformly higher on int8 -> inherent int8 quant/dequant overhead + (memory lever, not speed) -> memory-gate the auto-quant. +- erratic slow forwards / high recompile count / eager fallback -> a fixable compile inefficiency + (int8 tensor subclass breaks dynamic-shape compile) -> fix the compile path. + +Run: CUDA_VISIBLE_DEVICES=3 python scripts/hunyuan_int8_profile.py --modes dense,int8 +""" + +from __future__ import annotations + +import argparse +import os +import sys +import time +from pathlib import Path + +os.environ.setdefault("BITSANDBYTES_NOWELCOME", "1") +os.environ.setdefault("HF_HOME", "/mnt/disks/unslothai/ubuntu/workspace_81/BACKUP_05/temp/hf_cache") + +_REPO_ROOT = Path(__file__).resolve().parent.parent +for _p in (str(_REPO_ROOT / "studio" / "backend"), str(_REPO_ROOT / "scripts")): + if _p not in sys.path: + sys.path.insert(0, _p) + +from video_speedmem_bench import _build_pipe, _apply_levers, _target, PROMPT # noqa: E402 + +_REPO = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" + + +def _dynamo_counts(): + """(recompiles, graph_breaks, unique_compiles) from dynamo counters, best-effort.""" + try: + from torch._dynamo.utils import counters + rc = sum(v for k, v in counters.get("recompiles", {}).items()) if "recompiles" in counters else 0 + gb = sum(counters.get("graph_break", {}).values()) + # total frames compiled + stats = counters.get("stats", {}) + uc = stats.get("unique_graphs", 0) + return rc, gb, uc + except Exception: + return -1, -1, -1 + + +def _profile_mode(mode: str, *, steps: int, width: int, height: int, num_frames: int, guidance: float, + seed: int): + import torch + from core.inference.video_families import detect_video_family + from core.inference.diffusion_cache import maybe_toggle_step_cache # noqa: F401 + + # fresh dynamo state per mode + try: + torch._dynamo.reset() + from torch._dynamo.utils import counters + counters.clear() + except Exception: + pass + + fam_obj = detect_video_family(_REPO) + gvg = bool(getattr(fam_obj, "guidance_via_guider", False)) + default_steps = getattr(fam_obj, "default_steps", 50) + + pipe = _build_pipe(_REPO, False) + cfg = dict(te="none", vae="none", dit=("int8" if mode == "int8" else "none"), + speed="default", attn="native", cache="off") + engaged = _apply_levers(pipe, cfg, fam_name="hunyuanvideo-1.5", fam_obj=fam_obj, + force_fp32_vae=False, default_steps=default_steps) + print(f"[{mode}] dit_scheme={engaged['dit'] or 'dense'} speed={engaged.get('_effective_speed')}", + flush=True) + + # per-forward GPU timing via cuda events on the transformer. + fwd_ms: list[float] = [] + starts: list = [] + + def _pre(mod, args, kwargs): + ev = torch.cuda.Event(enable_timing=True) + ev.record() + starts.append(ev) + return None + + def _post(mod, args, output): + end = torch.cuda.Event(enable_timing=True) + end.record() + torch.cuda.synchronize() + if starts: + fwd_ms.append(starts[-1].elapsed_time(end)) + + h1 = pipe.transformer.register_forward_pre_hook(_pre, with_kwargs=True) + h2 = pipe.transformer.register_forward_hook(_post) + + def _gen(tag): + fwd_ms.clear() + starts.clear() + g = torch.Generator(device="cuda").manual_seed(seed) + kwargs = dict(prompt=PROMPT, width=width, height=height, num_frames=num_frames, + num_inference_steps=steps, generator=g) + if gvg: + guider = getattr(pipe, "guider", None) + if guider is not None and hasattr(guider, "guidance_scale"): + guider.guidance_scale = guidance + else: + kwargs["guidance_scale"] = guidance + torch.cuda.synchronize() + t0 = time.perf_counter() + pipe(**kwargs) + torch.cuda.synchronize() + total = time.perf_counter() - t0 + rc, gb, uc = _dynamo_counts() + n = len(fwd_ms) + srt = sorted(fwd_ms) + med = srt[n // 2] if n else 0.0 + p10 = srt[max(0, n // 10)] if n else 0.0 + p90 = srt[min(n - 1, (9 * n) // 10)] if n else 0.0 + slow = sum(1 for x in fwd_ms if x > 2.0 * med) if med else 0 + print(f"[{mode}:{tag}] total={total:.2f}s n_fwd={n} med={med:.1f}ms " + f"p10={p10:.1f} p90={p90:.1f} max={max(fwd_ms) if fwd_ms else 0:.1f} " + f"slow(>2x med)={slow} | recompiles={rc} graph_breaks={gb} unique_graphs={uc}", + flush=True) + return total + + _gen("warmup") # pays compile + _gen("timed") + _gen("timed2") + + h1.remove(); h2.remove() + del pipe + import gc + gc.collect() + torch.cuda.empty_cache() + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument("--modes", default="dense,int8") + ap.add_argument("--steps", type=int, default=30) + ap.add_argument("--num-frames", type=int, default=25) + ap.add_argument("--width", type=int, default=512) + ap.add_argument("--height", type=int, default=320) + ap.add_argument("--guidance", type=float, default=6.0) + ap.add_argument("--seed", type=int, default=42) + args = ap.parse_args(argv) + for mode in [m.strip() for m in args.modes.split(",") if m.strip()]: + _profile_mode(mode, steps=args.steps, width=args.width, height=args.height, + num_frames=args.num_frames, guidance=args.guidance, seed=args.seed) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/video_speedmem_bench.py b/scripts/video_speedmem_bench.py index 8c06928b83..5a9e33051e 100644 --- a/scripts/video_speedmem_bench.py +++ b/scripts/video_speedmem_bench.py @@ -444,6 +444,15 @@ def _run_config(name: str, cfg: dict, *, family: str, steps: int, width: int, he gvg = bool(getattr(fam_obj, "guidance_via_guider", False)) _empty(); _reset_peak() + # Fresh dynamo state per config so a prior config's compiled graphs cannot leak into this one + # (each config builds a fresh pipe; without this, later configs in a multi-config run can be + # measured against a dirty compile cache). + try: + import torch + + torch._dynamo.reset() + except Exception: + pass pipe = _build_pipe(repo, force_fp32) load_peak = _peak_gb() engaged = _apply_levers( diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py index cea2b4ad99..6ca1d39618 100644 --- a/studio/backend/core/inference/diffusion_transformer_quant.py +++ b/studio/backend/core/inference/diffusion_transformer_quant.py @@ -344,6 +344,28 @@ def select_transformer_quant_scheme( return None +def is_int8_memory_fallback(target: Any, family: Optional[str] = None) -> bool: + """Whether AUTO DiT quant would land on int8 only as a black-frame / denied FALLBACK on a + data-center, fp8-capable GPU -- i.e. fp8 would be the arch's pick but is denied/unavailable for + this family (e.g. HunyuanVideo-1.5). In that case int8 is a MEMORY lever, not a speed win: + measured on a B200 it is ~7% slower AND less accurate than the dense bf16 + regional-compile + path, so the loader prefers dense when the DiT fits resident (quantising only when memory- + constrained). Returns False where int8 is a legitimate accelerator and must be kept: + - consumer / workstation GPUs (fp8 FP32-accumulate is halved, so int8 is as fast or faster); + - pre-Ada data-center (sm < 8.9) with no fp8 tensor cores at all (int8 is the only quant); + - families whose auto scheme is fp8 (Wan / LTX) -- not int8, so nothing to prefer away from. + Best-effort: any probe failure returns False (keep today's behaviour).""" + try: + if _is_consumer_gpu(getattr(target, "device", None)): + return False + cap = _capability() + if cap is None or cap < (8, 9): # no fp8 tensor cores -> int8 is the genuine accelerator + return False + return select_transformer_quant_scheme(target, TQ_AUTO, family = family) == TQ_INT8 + except Exception: # noqa: BLE001 — optimisation gate only; never break the load + return False + + def _prefer_consumer_scheme(schemes: tuple[str, ...], device: Any) -> tuple[str, ...]: """Reorder an arch tier's schemes for the GPU class. On a consumer / workstation card move int8 to the front: consumer parts halve fp8/fp16 FP32-accumulate throughput, while diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py index 84d31decbd..3a181eaea6 100644 --- a/studio/backend/core/inference/video.py +++ b/studio/backend/core/inference/video.py @@ -74,6 +74,7 @@ from .diffusion_auto_policy import _QUANT_STEADY_FACTOR, build_resolved_record from .diffusion_transformer_quant import ( TQ_AUTO, dense_transformer_supported, + is_int8_memory_fallback, normalize_transformer_quant, quantize_transformer, select_transformer_quant_scheme, @@ -1150,7 +1151,28 @@ class VideoBackend: # so it runs before apply_speed_optims below -- same order as diffusion.py. transformer_quant_engaged: Optional[str] = None quant_skipped_for_offload = False - if ( + # Auto-quant lands on int8 for fp8-denied families (HunyuanVideo-1.5); measured on a B200, + # int8 is ~7% slower AND less accurate than the dense bf16 + regional-compile path (fp8, the + # only quant that also speeds up, is black-framed there). int8's sole benefit is memory, so + # when the DENSE DiT already fits resident (bf16_plan has no offload) prefer dense and skip + # the auto-quant. Only for an AUTO request (explicit int8/fp8 honored), only where int8 is a + # denied fallback on a data-center fp8-capable GPU (is_int8_memory_fallback excludes consumer + # / Ampere / fp8 families), and only when dense provably fits -- so no new OOM risk and the + # fp8 families (Wan / LTX) still quantise for their speed win. + quant_skipped_for_dense = ( + kind == "pipeline" + and normalize_transformer_quant(transformer_quant) == TQ_AUTO + and dense_transformer_supported(target) + and bf16_plan.offload_policy == "none" + and is_int8_memory_fallback(target, fam.name) + ) + if quant_skipped_for_dense: + logger.info( + "video.transformer_quant: skipped -- dense DiT fits resident and int8 (the fp8-denied " + "fallback for '%s') is slower + less accurate than dense+compile here; run dense", + fam.name, + ) + elif ( kind == "pipeline" and normalize_transformer_quant(transformer_quant) is not None and dense_transformer_supported(target) @@ -1403,6 +1425,9 @@ class VideoBackend: "skipped: offload moves the DiT, unsupported for torchao " "tensors; pin a resident memory mode to combine them" if quant_skipped_for_offload + else "skipped: dense DiT fits resident and int8 (fp8-denied fallback) " + "is slower + less accurate than dense+compile here" + if quant_skipped_for_dense else "not engaged (dense bf16 DiT loaded)" ), ), diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.py index ff4da40337..4d33fc7937 100644 --- a/studio/backend/tests/test_diffusion_transformer_quant.py +++ b/studio/backend/tests/test_diffusion_transformer_quant.py @@ -621,6 +621,35 @@ def test_family_deny_no_family_keeps_ladder(monkeypatch): assert select_transformer_quant_scheme(_target(), "auto", family = "sdxl") == TQ_FP8 +def test_is_int8_memory_fallback(monkeypatch): + # True only where AUTO lands on int8 as a black-frame/denied FALLBACK on a data-center, + # fp8-capable GPU (Hunyuan: fp8 denied -> int8), where dense+compile beats int8. The loader + # uses this (plus a 'dense fits resident' check) to prefer dense over int8. False everywhere + # int8 is a legitimate accelerator: fp8 families, consumer GPUs, and pre-Ada (no fp8) parts. + from core.inference.diffusion_transformer_quant import is_int8_memory_fallback + + # data-center Blackwell, all schemes available: Hunyuan denies fp8/mx/nvfp4 -> auto int8 -> True. + _stub_torch(monkeypatch, cc = (10, 0), device_name = "NVIDIA B200") + _allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8}) + assert is_int8_memory_fallback(_target(), "hunyuanvideo-1.5") is True + assert is_int8_memory_fallback(_target(), "hunyuanvideo-1.5-720p") is True + # Wan / LTX resolve to fp8 (a speed win), not int8 -> False (keep quantising). + assert is_int8_memory_fallback(_target(), "wan2.2-ti2v-5b") is False + assert is_int8_memory_fallback(_target(), "wan2.2-t2v-a14b") is False + assert is_int8_memory_fallback(_target(), "ltx-2") is False + assert is_int8_memory_fallback(_target(), None) is False + + # Consumer GPU (fp8 accumulate halved -> int8 can be a speed win): never prefer dense. + _stub_torch(monkeypatch, cc = (10, 0), device_name = "NVIDIA GeForce RTX 5090") + _allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8}) + assert is_int8_memory_fallback(_target(), "hunyuanvideo-1.5") is False + + # Ampere data-center (sm_80, no fp8 tensor cores): int8 is the genuine accelerator -> False. + _stub_torch(monkeypatch, cc = (8, 0), device_name = "NVIDIA A100") + _allow(monkeypatch, {TQ_INT8}) + assert is_int8_memory_fallback(_target(), "hunyuanvideo-1.5") is False + + def test_quantize_transformer_threads_family(monkeypatch): # quantize_transformer passes the family down to the selector, so a denied # (family, scheme) pair never reaches torchao.