diff --git a/scripts/video_quality.py b/scripts/video_quality.py new file mode 100644 index 0000000000..515700c370 --- /dev/null +++ b/scripts/video_quality.py @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Video quality-vs-cost harness for the Studio video backend. + +The video analogue of scripts/diffusion_quality.py: hold the prompt + seed + +shape fixed, render one clip with a high-fidelity reference configuration +(default the family's BF16 artifact), then render the same clip with each +candidate configuration (a GGUF quant, a dense torchao quant, a speed profile, +a step cache) and measure how far the output drifts from the reference. + +Per candidate it reports: +- mean PSNR / SSIM over evenly sampled frames (pixel + structural fidelity), +- a temporal-consistency deviation: the relative error between the reference's + and the candidate's frame-to-frame motion-energy series, which catches + flicker/juddering that per-frame SSIM alone can miss, +- black-frame and NaN collapse checks (the failure mode quant bugs actually + produce, per the image backend's qwen fp8 incident), +- an audio check for families that generate sound (LTX-2): RMS ratio vs the + reference and a silence trip-wire, +- wall time per generate and peak VRAM. + +Verdict bands map the standing accuracy budget: a candidate that keeps mean +SSIM at or above 0.75 PASSes (a ~25 percent structural drift is acceptable for +a large speed/memory win), 0.50-0.75 WARNs, and below 0.50 or any black/NaN/ +silence collapse FAILs regardless of how fast it is. + +Runtime-budgeted: ONE short clip per candidate (default 33 frames at 480p-class +sizes) so a full family sweep stays in minutes, not hours. Metrics are pure +numpy; torch / diffusers / the backend load lazily so --help and --selftest run +on a host without them. + +Examples: + # CPU metric sanity check (no GPU, no model): + python scripts/video_quality.py --selftest + + # LTX-2.3 GGUF quants against the BF16 GGUF reference: + CUDA_VISIBLE_DEVICES=1 python scripts/video_quality.py \\ + --model unsloth/LTX-2.3-GGUF --model-kind gguf \\ + --reference "gguf_filename=distilled-1.1/ltx-2.3-22b-distilled-1.1-BF16.gguf" \\ + --candidates "gguf_filename=distilled-1.1/ltx-2.3-22b-distilled-1.1-Q8_0.gguf" \\ + "gguf_filename=distilled-1.1/ltx-2.3-22b-distilled-1.1-UD-Q4_K_M.gguf" \\ + --steps 8 --guidance 1.0 --out-dir outputs/video_quality/ltx23 + + # Wan2.2-5B dense int8 + speed profiles against plain bf16: + CUDA_VISIBLE_DEVICES=1 python scripts/video_quality.py \\ + --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \\ + --reference "" \\ + --candidates "transformer_quant=int8" "speed_mode=max" \\ + --steps 20 --out-dir outputs/video_quality/wan5b +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys +import time +from pathlib import Path +from typing import Any, Optional + +_BACKEND_ROOT = Path(__file__).resolve().parent.parent / "studio" / "backend" +if str(_BACKEND_ROOT) not in sys.path: + sys.path.insert(0, str(_BACKEND_ROOT)) + +DEFAULT_PROMPT = ( + "a golden retriever puppy runs through shallow ocean waves at sunset, " + "splashing water, cinematic, camera tracking sideways" +) + +# Finite PSNR (dB) an identical clip is capped to when averaged, matching +# scripts/diffusion_quality.py. +_PERFECT_MATCH_PSNR = 100.0 + + +# ── frame metrics (pure numpy; frames are uint8 HxWx3 arrays) ──────────────── + + +def _gray(frame: Any) -> Any: + import numpy as np + + f = np.asarray(frame, dtype = np.float64) + return f @ np.array([0.299, 0.587, 0.114]) + + +def frame_psnr(a: Any, b: Any) -> float: + import numpy as np + + a64 = np.asarray(a, dtype = np.float64) + b64 = np.asarray(b, dtype = np.float64) + if a64.shape != b64.shape: + return 0.0 + mse = float(((a64 - b64) ** 2).mean()) + if mse == 0.0: + return math.inf + return 20.0 * math.log10(255.0) - 10.0 * math.log10(mse) + + +def _box_mean(x: Any, w: int) -> Any: + import numpy as np + + r = w // 2 + xp = np.pad(x, r, mode = "edge") + ii = np.cumsum(np.cumsum(xp, axis = 0), axis = 1) + ii = np.pad(ii, ((1, 0), (1, 0)), mode = "constant") + h, wd = x.shape + total = ii[w : h + w, w : wd + w] - ii[0:h, w : wd + w] - ii[w : h + w, 0:wd] + ii[0:h, 0:wd] + return total / float(w * w) + + +def frame_ssim(a: Any, b: Any, window: int = 7) -> float: + """Pure numpy box-window SSIM on luminance (Wang et al. constants); identical + math to scripts/diffusion_quality.py so image and video budgets compare.""" + ga, gb = _gray(a), _gray(b) + if ga.shape != gb.shape: + return 0.0 + c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2 + mu_a, mu_b = _box_mean(ga, window), _box_mean(gb, window) + mu_a2, mu_b2, mu_ab = mu_a * mu_a, mu_b * mu_b, mu_a * mu_b + var_a = _box_mean(ga * ga, window) - mu_a2 + var_b = _box_mean(gb * gb, window) - mu_b2 + cov_ab = _box_mean(ga * gb, window) - mu_ab + ssim_map = ((2 * mu_ab + c1) * (2 * cov_ab + c2)) / ( + (mu_a2 + mu_b2 + c1) * (var_a + var_b + c2) + ) + return float(ssim_map.mean()) + + +def motion_energy(frames: Any) -> list[float]: + """Mean absolute frame-to-frame luminance difference, one value per frame + transition. The temporal signature of the clip: flicker inflates it, frozen + or smeared motion deflates it.""" + import numpy as np + + grays = [_gray(f) for f in frames] + return [float(np.abs(grays[i + 1] - grays[i]).mean()) for i in range(len(grays) - 1)] + + +def temporal_deviation(ref_frames: Any, cand_frames: Any) -> float: + """Relative L1 error between the two motion-energy series (0 = identical + temporal behaviour). Series lengths must match (same frame count).""" + ref_series = motion_energy(ref_frames) + cand_series = motion_energy(cand_frames) + if len(ref_series) != len(cand_series) or not ref_series: + return math.inf + denom = sum(abs(v) for v in ref_series) + 1e-6 + return sum(abs(r - c) for r, c in zip(ref_series, cand_series)) / denom + + +def clip_metrics(ref_frames: Any, cand_frames: Any, sample_count: int = 5) -> dict[str, Any]: + """All frame metrics for one candidate clip vs the reference clip.""" + import numpy as np + + n = min(len(ref_frames), len(cand_frames)) + idx = sorted({int(round(i * (n - 1) / max(1, sample_count - 1))) for i in range(sample_count)}) + psnrs = [min(frame_psnr(ref_frames[i], cand_frames[i]), _PERFECT_MATCH_PSNR) for i in idx] + ssims = [frame_ssim(ref_frames[i], cand_frames[i]) for i in idx] + lumas = [float(_gray(cand_frames[i]).mean() / 255.0) for i in idx] + has_nan = any( + bool(np.isnan(np.asarray(f, dtype = np.float64)).any()) for f in (cand_frames[i] for i in idx) + ) + return { + "frames_compared": len(idx), + "psnr_mean": sum(psnrs) / len(psnrs), + "ssim_mean": sum(ssims) / len(ssims), + "temporal_deviation": temporal_deviation(ref_frames[:n], cand_frames[:n]), + "min_luma": min(lumas), + "has_nan": has_nan, + } + + +def audio_metrics(ref_audio: Optional[Any], cand_audio: Optional[Any]) -> dict[str, Any]: + """RMS comparison for families with sound. None audio on both sides is fine; + losing the track (or emitting silence) when the reference has one is not.""" + import numpy as np + + def _rms(a: Any) -> Optional[float]: + if a is None: + return None + arr = np.asarray(a, dtype = np.float64) + return float(np.sqrt((arr ** 2).mean())) if arr.size else 0.0 + + ref_rms, cand_rms = _rms(ref_audio), _rms(cand_audio) + silent_collapse = ( + ref_rms is not None and ref_rms >= 1e-3 and (cand_rms is None or cand_rms < 1e-4) + ) + return {"ref_rms": ref_rms, "cand_rms": cand_rms, "silent_collapse": silent_collapse} + + +def verdict(metrics: dict[str, Any], audio: dict[str, Any]) -> str: + """PASS / WARN / FAIL per the standing accuracy budget (~25 percent structural + drift acceptable, 50 percent or a collapse never).""" + if metrics["has_nan"] or metrics["min_luma"] < 0.02 or audio.get("silent_collapse"): + return "FAIL" + if metrics["ssim_mean"] < 0.50 or metrics["temporal_deviation"] > 1.0: + return "FAIL" + if metrics["ssim_mean"] < 0.75 or metrics["temporal_deviation"] > 0.5: + return "WARN" + return "PASS" + + +# ── mp4 decode (PyAV, same dependency the backend encodes with) ───────────── + + +def decode_mp4(mp4_bytes: bytes, workdir: Path, name: str) -> tuple[list[Any], Optional[Any]]: + """Frames (uint8 arrays) + mono audio samples (float array or None) from bytes.""" + import av + import numpy as np + + path = workdir / f"{name}.mp4" + path.write_bytes(mp4_bytes) + container = av.open(str(path)) + frames = [f.to_ndarray(format = "rgb24") for f in container.decode(container.streams.video[0])] + audio = None + if container.streams.audio: + container.close() + container = av.open(str(path)) + chunks = [c.to_ndarray() for c in container.decode(container.streams.audio[0])] + if chunks: + audio = np.concatenate([c.reshape(c.shape[0], -1).mean(axis = 0) for c in chunks]) + container.close() + return frames, audio + + +# ── configuration plumbing ─────────────────────────────────────────────────── + + +def parse_spec(spec: str) -> dict[str, str]: + """'k=v;k=v' (or space-free 'k=v,k=v') -> dict; empty string -> {} (pure base).""" + out: dict[str, str] = {} + for part in spec.replace(",", ";").split(";"): + part = part.strip() + if not part: + continue + if "=" not in part: + raise ValueError(f"Bad candidate spec fragment '{part}' (expected key=value)") + key, value = part.split("=", 1) + out[key.strip()] = value.strip() + return out + + +def spec_label(spec: dict[str, str]) -> str: + if not spec: + return "base" + return ",".join(f"{k}={Path(v).name if k == 'gguf_filename' else v}" for k, v in sorted(spec.items())) + + +def run_config(backend: Any, args: Any, spec: dict[str, str], workdir: Path, name: str) -> dict[str, Any]: + """Load per spec, generate the fixed clip, unload. Returns frames/audio/cost.""" + import torch + + load_kwargs: dict[str, Any] = { + "gguf_filename": spec.get("gguf_filename"), + "model_kind": spec.get("model_kind", args.model_kind), + "memory_mode": spec.get("memory_mode"), + "speed_mode": spec.get("speed_mode"), + "attention_backend": spec.get("attention_backend"), + "transformer_cache": spec.get("transformer_cache"), + "transformer_quant": spec.get("transformer_quant"), + } + t0 = time.monotonic() + status = backend.load_pipeline(args.model, **load_kwargs) + load_s = time.monotonic() - t0 + if torch.cuda.is_available(): + torch.cuda.reset_peak_memory_stats() + t0 = time.monotonic() + result = backend.generate( + prompt = args.prompt, + width = args.width, + height = args.height, + num_frames = args.frames, + fps = args.fps, + steps = args.steps, + guidance = args.guidance, + seed = args.seed, + ) + generate_s = time.monotonic() - t0 + peak_gib = ( + torch.cuda.max_memory_allocated() / 2**30 if torch.cuda.is_available() else 0.0 + ) + backend.unload() + frames, audio = decode_mp4(result["mp4_bytes"], workdir, name) + return { + "frames": frames, + "audio": audio, + "load_s": round(load_s, 1), + "generate_s": round(generate_s, 1), + "peak_vram_gib": round(peak_gib, 2), + "resolved": {k: v for k, v in status.items() if k in ( + "speed_mode", "attention_backend", "transformer_cache", "transformer_quant", + "offload_policy", "model_kind", + )}, + } + + +def run_gate(args: Any) -> int: + from core.inference.video import get_video_backend + + out_dir = Path(args.out_dir) + out_dir.mkdir(parents = True, exist_ok = True) + backend = get_video_backend() + + print(f"reference: {args.reference or 'base'}", flush = True) + ref = run_config(backend, args, parse_spec(args.reference), out_dir, "reference") + print( + f" load {ref['load_s']}s, generate {ref['generate_s']}s, " + f"peak {ref['peak_vram_gib']} GiB", + flush = True, + ) + + rows = [] + for spec_str in args.candidates: + spec = parse_spec(spec_str) + label = spec_label(spec) + print(f"candidate: {label}", flush = True) + cand = run_config(backend, args, spec, out_dir, label.replace("/", "_").replace("=", "-")) + metrics = clip_metrics(ref["frames"], cand["frames"], sample_count = args.sample_frames) + audio = audio_metrics(ref["audio"], cand["audio"]) + row = { + "candidate": label, + **{k: (round(v, 4) if isinstance(v, float) and math.isfinite(v) else v) + for k, v in metrics.items()}, + **{f"audio_{k}": v for k, v in audio.items()}, + "load_s": cand["load_s"], + "generate_s": cand["generate_s"], + "ref_generate_s": ref["generate_s"], + "peak_vram_gib": cand["peak_vram_gib"], + "resolved": cand["resolved"], + "verdict": verdict(metrics, audio), + } + rows.append(row) + print( + f" ssim {row['ssim_mean']:.3f} | psnr {row['psnr_mean']:.1f} dB | " + f"temporal {row['temporal_deviation']:.3f} | luma>={row['min_luma']:.3f} | " + f"gen {row['generate_s']}s (ref {ref['generate_s']}s) | " + f"vram {row['peak_vram_gib']} GiB | {row['verdict']}", + flush = True, + ) + + report = { + "model": args.model, + "reference": args.reference or "base", + "prompt": args.prompt, + "shape": [args.width, args.height, args.frames, args.fps], + "steps": args.steps, + "guidance": args.guidance, + "seed": args.seed, + "reference_cost": {k: ref[k] for k in ("load_s", "generate_s", "peak_vram_gib")}, + "candidates": rows, + } + (out_dir / "report.json").write_text(json.dumps(report, indent = 1)) + print(f"report: {out_dir / 'report.json'}", flush = True) + return 0 if all(r["verdict"] != "FAIL" for r in rows) else 1 + + +# ── selftest (CPU-only, synthetic clips, no torch/model) ──────────────────── + + +def selftest() -> int: + import numpy as np + + rng = np.random.default_rng(0) + h, w, n = 64, 96, 12 + + def make_clip(offset = 0.0, noise = 0.0, black = False): + frames = [] + for t in range(n): + x = np.linspace(0, 1, w)[None, :] + t * 0.05 + offset + base = (np.sin(x * 6.283) * 0.5 + 0.5) * 255.0 + frame = np.repeat(base[..., None], 3, axis = 2) * np.ones((h, 1, 1)) + if noise: + frame = frame + rng.normal(0, noise, frame.shape) + if black: + frame = frame * 0.0 + frames.append(np.clip(frame, 0, 255).astype(np.uint8)) + return frames + + ref = make_clip() + ok = True + + def check(cond, msg): + nonlocal ok + print(("PASS: " if cond else "FAIL: ") + msg) + ok = ok and cond + + same = clip_metrics(ref, make_clip()) + check(same["ssim_mean"] > 0.99 and same["temporal_deviation"] < 0.01, + f"identical clip scores ~1 (ssim {same['ssim_mean']:.3f})") + check(verdict(same, {"silent_collapse": False}) == "PASS", "identical clip verdict PASS") + + noisy = clip_metrics(ref, make_clip(noise = 12.0)) + check(0.3 < noisy["ssim_mean"] < 0.99, f"noisy clip degrades ssim ({noisy['ssim_mean']:.3f})") + + black = clip_metrics(ref, make_clip(black = True)) + check(verdict(black, {"silent_collapse": False}) == "FAIL", + f"black clip verdict FAIL (min_luma {black['min_luma']:.3f})") + + shifted = clip_metrics(ref, make_clip(offset = 0.5)) + check(shifted["ssim_mean"] < same["ssim_mean"], "content shift lowers ssim") + + audio = audio_metrics(np.sin(np.linspace(0, 100, 16000)), np.zeros(16000)) + check(audio["silent_collapse"] is True, "silent audio collapse detected") + audio_ok = audio_metrics(np.sin(np.linspace(0, 100, 16000)), + np.sin(np.linspace(0, 100, 16000)) * 0.8) + check(audio_ok["silent_collapse"] is False, "attenuated audio is not a collapse") + + print("VIDEO-QUALITY-SELFTEST", "PASS" if ok else "FAIL") + return 0 if ok else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description = __doc__.split("\n")[0]) + parser.add_argument("--selftest", action = "store_true", help = "CPU metric sanity check") + parser.add_argument("--model", help = "Repo id handed to the video backend") + parser.add_argument("--model-kind", default = None, help = "pipeline | gguf | single_file") + parser.add_argument("--reference", default = "", help = "Reference spec 'k=v;k=v' ('' = plain base load)") + parser.add_argument("--candidates", nargs = "+", default = [], help = "Candidate specs 'k=v;k=v'") + parser.add_argument("--prompt", default = DEFAULT_PROMPT) + parser.add_argument("--width", type = int, default = 768) + parser.add_argument("--height", type = int, default = 512) + parser.add_argument("--frames", type = int, default = 33) + parser.add_argument("--fps", type = int, default = 24) + parser.add_argument("--steps", type = int, default = None) + parser.add_argument("--guidance", type = float, default = None) + parser.add_argument("--seed", type = int, default = 7) + parser.add_argument("--sample-frames", type = int, default = 5) + parser.add_argument("--out-dir", default = "outputs/video_quality") + args = parser.parse_args() + + if args.selftest: + return selftest() + if not args.model or not args.candidates: + parser.error("--model and --candidates are required (or use --selftest)") + return run_gate(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py index 9f5c5c76e6..b801deb609 100644 --- a/studio/backend/core/inference/video.py +++ b/studio/backend/core/inference/video.py @@ -30,6 +30,7 @@ family's official base repos, or a local path the user explicitly picked. from __future__ import annotations +import contextlib import inspect import os import tempfile @@ -100,6 +101,11 @@ _TRUSTED_NON_GGUF_VIDEO_REPOS = frozenset( # remote code, so allowed as full (pipeline-kind) loads like the LTX-2 bases. "wan-ai/wan2.2-ti2v-5b-diffusers", "wan-ai/wan2.2-t2v-a14b-diffusers", + # HunyuanVideo-1.5 community Diffusers repacks (tencent's own repo is the + # original non-diffusers layout: config.json, no model_index.json, so it + # cannot load through HunyuanVideo15Pipeline at all). + "hunyuanvideo-community/hunyuanvideo-1.5-diffusers-480p_t2v", + "hunyuanvideo-community/hunyuanvideo-1.5-diffusers-720p_t2v", } ) @@ -129,6 +135,37 @@ def _is_trusted_video_repo(repo_id: str) -> bool: return rid.startswith("unsloth/") or rid in _TRUSTED_NON_GGUF_VIDEO_REPOS +class _VideoGenerationCancelled(Exception): + """Unwinds a denoise loop that has no cooperative interrupt (no step callback); + generate() maps it to the VIDEO_CANCELLED_MSG sentinel the routes 409 on.""" + + +@contextlib.contextmanager +def _scheduler_step_progress(pipe: Any, on_step: Any): + """Progress + cancellation for pipelines WITHOUT callback_on_step_end. + + HunyuanVideo15Pipeline exposes no per-step callback, but every denoise step + makes exactly one ``scheduler.step`` call, so wrapping that method gives the + same per-step tick the callback path gets. ``on_step`` receives the 1-based + step count and may raise (_VideoGenerationCancelled) to abort the loop. The + original method is always restored, even when the pipeline raises. + """ + scheduler = pipe.scheduler + original = scheduler.step + count = {"n": 0} + + def _step(*args: Any, **kwargs: Any) -> Any: + count["n"] += 1 + on_step(count["n"]) + return original(*args, **kwargs) + + scheduler.step = _step + try: + yield + finally: + scheduler.step = original + + def _ensure_mp4_encoder_available() -> None: """Fail a load fast when PyAV is missing: the export otherwise dies AFTER a multi-minute denoise, which is the worst possible time to learn about it.""" @@ -835,12 +872,19 @@ class VideoBackend: kwargs: dict[str, Any] = { "prompt": prompt, "num_inference_steps": steps, - fam.cfg_kwarg: guidance, "width": width, "height": height, "num_frames": frames, "generator": generator, } + if fam.guidance_via_guider: + # HunyuanVideo-1.5: __call__ has no guidance kwarg at all; the + # CFG scale is a plain attribute on the pipeline's guider + # component, set per request. Near-1 scales auto-disable CFG + # inside the guider itself (_is_cfg_enabled's is_close check). + pipe.guider.guidance_scale = float(guidance) + else: + kwargs[fam.cfg_kwarg] = guidance if negative_prompt and "negative_prompt" in call_params: kwargs["negative_prompt"] = negative_prompt # LTX-2 takes frame_rate (it shapes the audio track length); other @@ -869,23 +913,42 @@ class VideoBackend: "started": started, "eta_seconds": None, "error": None, } - def _on_step(p, step_index, timestep, callback_kwargs): - if cancel.is_set(): - p._interrupt = True - return callback_kwargs - done = step_index + 1 + def _tick(done: int) -> None: elapsed = time.monotonic() - started self._gen.update( step = done, eta_seconds = (elapsed / max(1, done)) * max(0, steps - done), ) + + def _on_step(p, step_index, timestep, callback_kwargs): + if cancel.is_set(): + p._interrupt = True + return callback_kwargs + _tick(step_index + 1) return callback_kwargs + def _on_scheduler_step(done: int) -> None: + # No cooperative _interrupt here: without a callback the pipeline + # never checks it, so cancellation must unwind the denoise loop + # via an exception (mapped to the cancelled sentinel below). + if cancel.is_set(): + raise _VideoGenerationCancelled() + _tick(done) + if "callback_on_step_end" in call_params: kwargs["callback_on_step_end"] = _on_step + progress_ctx = contextlib.nullcontext() + else: + # HunyuanVideo-1.5 has no step callback; every scheduler.step + # call is exactly one denoise step, so wrap it for progress + + # cancel and restore it afterwards. + progress_ctx = _scheduler_step_progress(pipe, _on_scheduler_step) - with torch.inference_mode(): - output = pipe(**kwargs) + try: + with torch.inference_mode(), progress_ctx: + output = pipe(**kwargs) + except _VideoGenerationCancelled: + raise RuntimeError(VIDEO_CANCELLED_MSG) from None if cancel.is_set(): raise RuntimeError(VIDEO_CANCELLED_MSG) diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py index fde4947d0e..287321402b 100644 --- a/studio/backend/core/inference/video_families.py +++ b/studio/backend/core/inference/video_families.py @@ -50,6 +50,12 @@ class VideoFamily: transformer2_class: Optional[str] = None is_moe: bool = False cfg2_kwarg: Optional[str] = None + # HunyuanVideo-1.5 style guidance: the pipeline __call__ takes NO guidance + # kwarg at all; CFG lives on a ``guider`` component (ClassifierFreeGuidance) + # whose ``guidance_scale`` is a plain attribute set per request. When True, + # generate() writes the scale onto ``pipe.guider`` instead of passing + # ``cfg_kwarg`` (which the pipeline would reject as an unexpected argument). + guidance_via_guider: bool = False # Generation defaults + shape constraints. ``frame_step`` is the temporal # compression: a valid frame count is k * frame_step + 1 (the +1 is the # anchor frame), so requests are snapped BEFORE latents are allocated. @@ -190,6 +196,46 @@ _FAMILIES: tuple[VideoFamily, ...] = ( bf16_components_gb = (114.3, 11.4, 0.5), gguf_repo = "QuantStack/Wan2.2-T2V-A14B-GGUF", ), + # HunyuanVideo-1.5 (diffusers >= 0.39): an 8.3B video DiT with a Qwen2.5-VL + # text encoder plus a ByT5 glyph encoder. Three quirks, all verified against + # the installed pipeline source (pipeline_hunyuan_video1_5.py): + # 1. __call__ takes NO guidance kwarg; CFG lives on the ``guider`` component + # (ClassifierFreeGuidance; the 480p t2v repo ships guidance_scale = 6.0), + # hence guidance_via_guider. + # 2. __call__ has NO callback_on_step_end; generate() falls back to the + # scheduler.step progress wrapper automatically (capability-detected). + # 3. The tencent/HunyuanVideo-1.5 repo is the ORIGINAL layout (config.json, + # no model_index.json); only the hunyuanvideo-community Diffusers repacks + # load through HunyuanVideo15Pipeline, so those are the trusted repos. + # The transformer declares _repeated_blocks and inherits CacheMixin, so the + # regional compile profile and First-Block-Cache both apply. + VideoFamily( + name = "hunyuanvideo-1.5", + pipeline_class = "HunyuanVideo15Pipeline", + transformer_class = "HunyuanVideo15Transformer3DModel", + base_repo = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + # No bare "hunyuanvideo" alias: it would also claim the incompatible 1.0 + # repos (HunyuanVideoPipeline), which this family cannot load. + aliases = ("hunyuanvideo-1-5", "hunyuanvideo1.5", "hunyuanvideo1-5", "hv15"), + has_audio = False, + guidance_via_guider = True, + default_steps = 50, + default_guidance = 6.0, + # 121 frames at 24 fps is ~5s, the pipeline's own num_frames default. + default_num_frames = 121, + default_fps = 24, + # The HV15 VAE compresses 16x spatial / 4x temporal (vae config: + # spatial_compression_ratio 16, temporal_compression_ratio 4) with a + # patch-1 transformer, so sizes snap to /16 and frames to 4k+1. + frame_step = 4, + resolution_multiple = 16, + # 480p-class presets (the base repo is the 480p t2v variant): landscape, + # vertical, square. + resolution_presets = ((832, 480), (480, 832), (624, 624)), + # Disk shards are fp32 for the DiT (32.0 GB -> 16.6 bf16-resident) and the + # VAE (4.7 -> 2.4); the Qwen2.5-VL TE is stored bf16 (14.0) plus ByT5 0.8. + bf16_components_gb = (16.6, 14.8, 2.4), + ), ) @@ -262,6 +308,9 @@ _VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = ( # and the base repo. A future distilled Wan GGUF is caught by the "distilled" # row above (listed first), exactly as the LTX-2.3 distilled checkpoints are. ("wan", 50, 5.0), + # HunyuanVideo-1.5 runs the pipeline's 50 steps with the guider's shipped + # CFG 6.0 (guider_config.json in the community Diffusers repacks). + ("hunyuanvideo", 50, 6.0), ) diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py index c1e461ec8a..4bca9f3914 100644 --- a/studio/backend/tests/test_video_backend.py +++ b/studio/backend/tests/test_video_backend.py @@ -13,7 +13,7 @@ import types import pytest from core.inference.video import VideoBackend, get_video_backend, resolve_video_model_kind -from core.inference.video_families import VIDEO_NOT_LOADED_MSG +from core.inference.video_families import VIDEO_CANCELLED_MSG, VIDEO_NOT_LOADED_MSG class _FakeDtype: @@ -273,6 +273,82 @@ class _FakeWanPipelineSingle: return _FakeWanPipeMoE() if moe else _FakeWanPipeSingle() +# ── HunyuanVideo-1.5 fakes: the __call__ signature has NO guidance kwarg and NO +# callback_on_step_end (matching pipeline_hunyuan_video1_5.py in diffusers 0.39), +# a guider object carries the CFG scale, and the denoise loop drives +# scheduler.step -- so the guider write and the scheduler-wrap progress/cancel +# paths actually exercise. + + +class _FakeHV15Scheduler: + def __init__(self) -> None: + self.calls = 0 + # Test hook fired from the ORIGINAL step (i.e. inside the wrapped call), + # letting a test cancel mid-denoise exactly as a user request would land. + self.on_step = None + + def step(self, *args, **kwargs): + self.calls += 1 + if self.on_step is not None: + self.on_step(self.calls) + return object() + + +class _FakeHV15Pipe: + def __init__(self) -> None: + self.vae = _FakeWanVae() + self.transformer = _FakeWanDiT() + self.scheduler = _FakeHV15Scheduler() + self.guider = types.SimpleNamespace(guidance_scale = 6.0) + self.components = {"transformer": self.transformer, "vae": self.vae} + self.moved_to = None + self.last_kwargs = None + + def to(self, device): + self.moved_to = device + return self + + def enable_vae_tiling(self) -> None: + self.vae.tiled = True + + def __call__( + self, + *, + prompt = None, + negative_prompt = None, + height = None, + width = None, + num_frames = None, + num_inference_steps = None, + generator = None, + **kwargs, + ): + self.last_kwargs = { + "prompt": prompt, + "negative_prompt": negative_prompt, + "num_inference_steps": num_inference_steps, + "width": width, + "height": height, + "num_frames": num_frames, + **kwargs, + } + for _ in range(int(num_inference_steps or 1)): + self.scheduler.step() + frames = [[object() for _ in range(int(num_frames or 1))]] + return types.SimpleNamespace(frames = frames, audio = None) + + +class _FakeHV15Pipeline: + last: dict = {} + instance = None + + @classmethod + def from_pretrained(cls, repo, **kwargs): + _FakeHV15Pipeline.last = {"repo": repo, **kwargs} + _FakeHV15Pipeline.instance = _FakeHV15Pipe() + return _FakeHV15Pipeline.instance + + @pytest.fixture def fake_runtime(monkeypatch): torch = types.ModuleType("torch") @@ -291,6 +367,8 @@ def fake_runtime(monkeypatch): # Wan2.2: one pipeline class serves both families (it dispatches on the repo id). diffusers.WanPipeline = _FakeWanPipelineSingle diffusers.WanTransformer3DModel = _FakeTransformer + diffusers.HunyuanVideo15Pipeline = _FakeHV15Pipeline + diffusers.HunyuanVideo15Transformer3DModel = _FakeTransformer diffusers.FirstBlockCacheConfig = lambda threshold = None: ("fbcache", threshold) monkeypatch.setitem(sys.modules, "torch", torch) @@ -500,6 +578,50 @@ def test_generate_progress_and_cancel_idle(fake_runtime): assert backend.cancel_generate() is False +def test_hv15_guider_and_scheduler_progress(fake_runtime): + # HunyuanVideo-1.5: no guidance kwarg (CFG set on the guider), no step + # callback (progress via the scheduler.step wrapper, restored afterwards). + backend = VideoBackend() + status = backend.load_pipeline( + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + model_kind = "pipeline", + ) + assert status["family"] == "hunyuanvideo-1.5" + assert status["has_audio"] is False + assert status["defaults"]["frame_step"] == 4 + + pipe = _FakeHV15Pipeline.instance + result = backend.generate( + prompt = "a fox in the snow", steps = 4, guidance = 3.5, num_frames = 9, fps = 24 + ) + assert "guidance_scale" not in pipe.last_kwargs + assert "callback_on_step_end" not in pipe.last_kwargs + assert pipe.guider.guidance_scale == 3.5 + # One wrapped tick per denoise step, then the original method back in place. + assert pipe.scheduler.calls == 4 + assert pipe.scheduler.step.__func__ is _FakeHV15Scheduler.step + assert result["num_frames"] == 9 and result["has_audio"] is False + + +def test_hv15_cancel_unwinds_scheduler_loop(fake_runtime): + backend = VideoBackend() + backend.load_pipeline( + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + model_kind = "pipeline", + ) + pipe = _FakeHV15Pipeline.instance + # Cancel lands during the FIRST real step; the next wrapped call must raise out + # of the denoise loop and generate() must surface the cancelled sentinel. + pipe.scheduler.on_step = ( + lambda n: backend.cancel_generate() if n == 1 else None + ) + with pytest.raises(RuntimeError, match = VIDEO_CANCELLED_MSG): + backend.generate(prompt = "a fox", steps = 4) + assert pipe.scheduler.calls == 1 + # The wrapper must restore scheduler.step even on the exception path. + assert pipe.scheduler.step.__func__ is _FakeHV15Scheduler.step + + def test_singleton(): assert get_video_backend() is get_video_backend() diff --git a/studio/backend/tests/test_video_families.py b/studio/backend/tests/test_video_families.py index 252699e3bb..0f816d1a29 100644 --- a/studio/backend/tests/test_video_families.py +++ b/studio/backend/tests/test_video_families.py @@ -150,6 +150,7 @@ def test_supported_names(): "ltx-2", "wan2.2-ti2v-5b", "wan2.2-t2v-a14b", + "hunyuanvideo-1.5", ) @@ -200,3 +201,25 @@ def test_family_size_table_present(): # that would let auto planning under-reserve by ~50 GB. assert text_encoder_gb > transformer_gb > 20.0 assert companions_gb > 0.0 + + +def test_hv15_detection_and_flags(): + fam = detect_video_family("hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v") + assert fam is not None and fam.name == "hunyuanvideo-1.5" + # CFG lives on the guider component (no guidance kwarg in __call__), and the + # HV15 VAE compresses 16x spatial / 4x temporal. + assert fam.guidance_via_guider is True + assert fam.frame_step == 4 and fam.resolution_multiple == 16 + assert fam.has_audio is False + assert detect_video_family("x/y", override = "hv15") is fam + # The incompatible HunyuanVideo 1.0 repos must NOT be claimed: their + # model_index pins HunyuanVideoPipeline, which this family cannot load. + assert detect_video_family("hunyuanvideo-community/HunyuanVideo") is None + + +def test_hv15_generation_defaults(): + # The community repacks ship a guider with guidance_scale 6.0 and the + # pipeline's own 50-step schedule. + assert default_video_generation_params( + None, "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v" + ) == (50, 6.0) diff --git a/studio/frontend/src/features/video/video-page.tsx b/studio/frontend/src/features/video/video-page.tsx index c9dbfd231d..1b401d3d28 100644 --- a/studio/frontend/src/features/video/video-page.tsx +++ b/studio/frontend/src/features/video/video-page.tsx @@ -72,6 +72,9 @@ const PIPELINE_MODELS: Record = { // these to the Wan-AI base repos (see _TRUSTED_NON_GGUF_VIDEO_REPOS). "Wan-AI/Wan2.2-TI2V-5B-Diffusers": { kind: "pipeline" }, "Wan-AI/Wan2.2-T2V-A14B-Diffusers": { kind: "pipeline" }, + // HunyuanVideo-1.5 community Diffusers repack (tencent's own repo is the original + // non-diffusers layout and cannot load as a pipeline). + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v": { kind: "pipeline" }, }; // A curated GGUF picker entry: isGguf true expands its .gguf files in the quant expander @@ -106,6 +109,11 @@ const VIDEO_MODELS: ModelOption[] = [ "Wan 2.2 T2V A14B (MoE)", "Text-to-video, dual-expert · Safetensors", ), + pipelineModel( + "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", + "HunyuanVideo 1.5 (480p)", + "Text-to-video 480p · Safetensors", + ), ]; // Per-model generation defaults (steps + guidance), matched by repo-id substring, most @@ -120,6 +128,9 @@ const MODEL_DEFAULTS: Array<{ match: string; steps: number; guidance: number }> // Wan2.2 pipelines default to 50 steps at CFG 5.0 (WanPipeline defaults, verified in // diffusers 0.39). The backend supplies the fps per family (24 for TI2V-5B, 16 for A14B). { match: "wan", steps: 50, guidance: 5 }, + // HunyuanVideo-1.5 runs 50 steps; guidance 6 matches the guider the repo ships + // (the backend writes it onto the guider component, there is no pipeline kwarg). + { match: "hunyuanvideo", steps: 50, guidance: 6 }, ]; function defaultsFor(repoId: string): { steps: number; guidance: number } {