Add HunyuanVideo-1.5 family and the video quality gate
HunyuanVideo-1.5 (the 8.3B DiT with Qwen2.5-VL + ByT5 text encoders) loads through the hunyuanvideo-community Diffusers repacks; tencent's own repo is the original non-diffusers layout and cannot load as a pipeline, so only the community 480p/720p t2v repos are trusted. Two pipeline quirks, both verified against pipeline_hunyuan_video1_5.py in diffusers 0.39, shape the wiring: - __call__ takes no guidance kwarg: CFG lives on the pipeline's guider component (ClassifierFreeGuidance, shipped at scale 6.0). The family gains guidance_via_guider and generate() writes the requested scale onto pipe.guider instead of passing cfg_kwarg, which the pipeline would reject. - __call__ has no callback_on_step_end: progress and cancellation fall back to a scheduler.step wrapper (one call per denoise step), installed for the duration of the call and always restored. Cancellation unwinds the loop by raising through the wrapper and surfaces the same cancelled sentinel the callback path uses. The VAE compresses 16x spatial / 4x temporal, so sizes snap to /16 and frame counts to 4k+1. The transformer declares _repeated_blocks and CacheMixin, so the regional compile profile and the step cache both apply unchanged. scripts/video_quality.py is the video accuracy gate, the analogue of scripts/diffusion_quality.py with the same pure-numpy PSNR/SSIM math so image and video budgets compare: fixed prompt/seed/shape, one short clip per candidate against a reference clip, per-frame SSIM/PSNR over sampled frames, a temporal-consistency deviation (motion-energy series error, catching flicker SSIM alone misses), black-frame/NaN collapse checks, an audio RMS silence trip-wire for LTX-2, and wall time + peak VRAM per candidate. Verdicts map the standing budget: ssim >= 0.75 passes, >= 0.50 warns, anything lower or any collapse fails. --selftest runs the metric path on synthetic clips with no GPU or model.
This commit is contained in:
Daniel Han
parent
e1221f3a47
commit
f77fa80007
6 changed files with 717 additions and 10 deletions
439
scripts/video_quality.py
Normal file
439
scripts/video_quality.py
Normal file
|
|
@ -0,0 +1,439 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Video quality-vs-cost harness for the Studio video backend.
|
||||
|
||||
The video analogue of scripts/diffusion_quality.py: hold the prompt + seed +
|
||||
shape fixed, render one clip with a high-fidelity reference configuration
|
||||
(default the family's BF16 artifact), then render the same clip with each
|
||||
candidate configuration (a GGUF quant, a dense torchao quant, a speed profile,
|
||||
a step cache) and measure how far the output drifts from the reference.
|
||||
|
||||
Per candidate it reports:
|
||||
- mean PSNR / SSIM over evenly sampled frames (pixel + structural fidelity),
|
||||
- a temporal-consistency deviation: the relative error between the reference's
|
||||
and the candidate's frame-to-frame motion-energy series, which catches
|
||||
flicker/juddering that per-frame SSIM alone can miss,
|
||||
- black-frame and NaN collapse checks (the failure mode quant bugs actually
|
||||
produce, per the image backend's qwen fp8 incident),
|
||||
- an audio check for families that generate sound (LTX-2): RMS ratio vs the
|
||||
reference and a silence trip-wire,
|
||||
- wall time per generate and peak VRAM.
|
||||
|
||||
Verdict bands map the standing accuracy budget: a candidate that keeps mean
|
||||
SSIM at or above 0.75 PASSes (a ~25 percent structural drift is acceptable for
|
||||
a large speed/memory win), 0.50-0.75 WARNs, and below 0.50 or any black/NaN/
|
||||
silence collapse FAILs regardless of how fast it is.
|
||||
|
||||
Runtime-budgeted: ONE short clip per candidate (default 33 frames at 480p-class
|
||||
sizes) so a full family sweep stays in minutes, not hours. Metrics are pure
|
||||
numpy; torch / diffusers / the backend load lazily so --help and --selftest run
|
||||
on a host without them.
|
||||
|
||||
Examples:
|
||||
# CPU metric sanity check (no GPU, no model):
|
||||
python scripts/video_quality.py --selftest
|
||||
|
||||
# LTX-2.3 GGUF quants against the BF16 GGUF reference:
|
||||
CUDA_VISIBLE_DEVICES=1 python scripts/video_quality.py \\
|
||||
--model unsloth/LTX-2.3-GGUF --model-kind gguf \\
|
||||
--reference "gguf_filename=distilled-1.1/ltx-2.3-22b-distilled-1.1-BF16.gguf" \\
|
||||
--candidates "gguf_filename=distilled-1.1/ltx-2.3-22b-distilled-1.1-Q8_0.gguf" \\
|
||||
"gguf_filename=distilled-1.1/ltx-2.3-22b-distilled-1.1-UD-Q4_K_M.gguf" \\
|
||||
--steps 8 --guidance 1.0 --out-dir outputs/video_quality/ltx23
|
||||
|
||||
# Wan2.2-5B dense int8 + speed profiles against plain bf16:
|
||||
CUDA_VISIBLE_DEVICES=1 python scripts/video_quality.py \\
|
||||
--model Wan-AI/Wan2.2-TI2V-5B-Diffusers \\
|
||||
--reference "" \\
|
||||
--candidates "transformer_quant=int8" "speed_mode=max" \\
|
||||
--steps 20 --out-dir outputs/video_quality/wan5b
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
_BACKEND_ROOT = Path(__file__).resolve().parent.parent / "studio" / "backend"
|
||||
if str(_BACKEND_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_BACKEND_ROOT))
|
||||
|
||||
DEFAULT_PROMPT = (
|
||||
"a golden retriever puppy runs through shallow ocean waves at sunset, "
|
||||
"splashing water, cinematic, camera tracking sideways"
|
||||
)
|
||||
|
||||
# Finite PSNR (dB) an identical clip is capped to when averaged, matching
|
||||
# scripts/diffusion_quality.py.
|
||||
_PERFECT_MATCH_PSNR = 100.0
|
||||
|
||||
|
||||
# ── frame metrics (pure numpy; frames are uint8 HxWx3 arrays) ────────────────
|
||||
|
||||
|
||||
def _gray(frame: Any) -> Any:
|
||||
import numpy as np
|
||||
|
||||
f = np.asarray(frame, dtype = np.float64)
|
||||
return f @ np.array([0.299, 0.587, 0.114])
|
||||
|
||||
|
||||
def frame_psnr(a: Any, b: Any) -> float:
|
||||
import numpy as np
|
||||
|
||||
a64 = np.asarray(a, dtype = np.float64)
|
||||
b64 = np.asarray(b, dtype = np.float64)
|
||||
if a64.shape != b64.shape:
|
||||
return 0.0
|
||||
mse = float(((a64 - b64) ** 2).mean())
|
||||
if mse == 0.0:
|
||||
return math.inf
|
||||
return 20.0 * math.log10(255.0) - 10.0 * math.log10(mse)
|
||||
|
||||
|
||||
def _box_mean(x: Any, w: int) -> Any:
|
||||
import numpy as np
|
||||
|
||||
r = w // 2
|
||||
xp = np.pad(x, r, mode = "edge")
|
||||
ii = np.cumsum(np.cumsum(xp, axis = 0), axis = 1)
|
||||
ii = np.pad(ii, ((1, 0), (1, 0)), mode = "constant")
|
||||
h, wd = x.shape
|
||||
total = ii[w : h + w, w : wd + w] - ii[0:h, w : wd + w] - ii[w : h + w, 0:wd] + ii[0:h, 0:wd]
|
||||
return total / float(w * w)
|
||||
|
||||
|
||||
def frame_ssim(a: Any, b: Any, window: int = 7) -> float:
|
||||
"""Pure numpy box-window SSIM on luminance (Wang et al. constants); identical
|
||||
math to scripts/diffusion_quality.py so image and video budgets compare."""
|
||||
ga, gb = _gray(a), _gray(b)
|
||||
if ga.shape != gb.shape:
|
||||
return 0.0
|
||||
c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2
|
||||
mu_a, mu_b = _box_mean(ga, window), _box_mean(gb, window)
|
||||
mu_a2, mu_b2, mu_ab = mu_a * mu_a, mu_b * mu_b, mu_a * mu_b
|
||||
var_a = _box_mean(ga * ga, window) - mu_a2
|
||||
var_b = _box_mean(gb * gb, window) - mu_b2
|
||||
cov_ab = _box_mean(ga * gb, window) - mu_ab
|
||||
ssim_map = ((2 * mu_ab + c1) * (2 * cov_ab + c2)) / (
|
||||
(mu_a2 + mu_b2 + c1) * (var_a + var_b + c2)
|
||||
)
|
||||
return float(ssim_map.mean())
|
||||
|
||||
|
||||
def motion_energy(frames: Any) -> list[float]:
|
||||
"""Mean absolute frame-to-frame luminance difference, one value per frame
|
||||
transition. The temporal signature of the clip: flicker inflates it, frozen
|
||||
or smeared motion deflates it."""
|
||||
import numpy as np
|
||||
|
||||
grays = [_gray(f) for f in frames]
|
||||
return [float(np.abs(grays[i + 1] - grays[i]).mean()) for i in range(len(grays) - 1)]
|
||||
|
||||
|
||||
def temporal_deviation(ref_frames: Any, cand_frames: Any) -> float:
|
||||
"""Relative L1 error between the two motion-energy series (0 = identical
|
||||
temporal behaviour). Series lengths must match (same frame count)."""
|
||||
ref_series = motion_energy(ref_frames)
|
||||
cand_series = motion_energy(cand_frames)
|
||||
if len(ref_series) != len(cand_series) or not ref_series:
|
||||
return math.inf
|
||||
denom = sum(abs(v) for v in ref_series) + 1e-6
|
||||
return sum(abs(r - c) for r, c in zip(ref_series, cand_series)) / denom
|
||||
|
||||
|
||||
def clip_metrics(ref_frames: Any, cand_frames: Any, sample_count: int = 5) -> dict[str, Any]:
|
||||
"""All frame metrics for one candidate clip vs the reference clip."""
|
||||
import numpy as np
|
||||
|
||||
n = min(len(ref_frames), len(cand_frames))
|
||||
idx = sorted({int(round(i * (n - 1) / max(1, sample_count - 1))) for i in range(sample_count)})
|
||||
psnrs = [min(frame_psnr(ref_frames[i], cand_frames[i]), _PERFECT_MATCH_PSNR) for i in idx]
|
||||
ssims = [frame_ssim(ref_frames[i], cand_frames[i]) for i in idx]
|
||||
lumas = [float(_gray(cand_frames[i]).mean() / 255.0) for i in idx]
|
||||
has_nan = any(
|
||||
bool(np.isnan(np.asarray(f, dtype = np.float64)).any()) for f in (cand_frames[i] for i in idx)
|
||||
)
|
||||
return {
|
||||
"frames_compared": len(idx),
|
||||
"psnr_mean": sum(psnrs) / len(psnrs),
|
||||
"ssim_mean": sum(ssims) / len(ssims),
|
||||
"temporal_deviation": temporal_deviation(ref_frames[:n], cand_frames[:n]),
|
||||
"min_luma": min(lumas),
|
||||
"has_nan": has_nan,
|
||||
}
|
||||
|
||||
|
||||
def audio_metrics(ref_audio: Optional[Any], cand_audio: Optional[Any]) -> dict[str, Any]:
|
||||
"""RMS comparison for families with sound. None audio on both sides is fine;
|
||||
losing the track (or emitting silence) when the reference has one is not."""
|
||||
import numpy as np
|
||||
|
||||
def _rms(a: Any) -> Optional[float]:
|
||||
if a is None:
|
||||
return None
|
||||
arr = np.asarray(a, dtype = np.float64)
|
||||
return float(np.sqrt((arr ** 2).mean())) if arr.size else 0.0
|
||||
|
||||
ref_rms, cand_rms = _rms(ref_audio), _rms(cand_audio)
|
||||
silent_collapse = (
|
||||
ref_rms is not None and ref_rms >= 1e-3 and (cand_rms is None or cand_rms < 1e-4)
|
||||
)
|
||||
return {"ref_rms": ref_rms, "cand_rms": cand_rms, "silent_collapse": silent_collapse}
|
||||
|
||||
|
||||
def verdict(metrics: dict[str, Any], audio: dict[str, Any]) -> str:
|
||||
"""PASS / WARN / FAIL per the standing accuracy budget (~25 percent structural
|
||||
drift acceptable, 50 percent or a collapse never)."""
|
||||
if metrics["has_nan"] or metrics["min_luma"] < 0.02 or audio.get("silent_collapse"):
|
||||
return "FAIL"
|
||||
if metrics["ssim_mean"] < 0.50 or metrics["temporal_deviation"] > 1.0:
|
||||
return "FAIL"
|
||||
if metrics["ssim_mean"] < 0.75 or metrics["temporal_deviation"] > 0.5:
|
||||
return "WARN"
|
||||
return "PASS"
|
||||
|
||||
|
||||
# ── mp4 decode (PyAV, same dependency the backend encodes with) ─────────────
|
||||
|
||||
|
||||
def decode_mp4(mp4_bytes: bytes, workdir: Path, name: str) -> tuple[list[Any], Optional[Any]]:
|
||||
"""Frames (uint8 arrays) + mono audio samples (float array or None) from bytes."""
|
||||
import av
|
||||
import numpy as np
|
||||
|
||||
path = workdir / f"{name}.mp4"
|
||||
path.write_bytes(mp4_bytes)
|
||||
container = av.open(str(path))
|
||||
frames = [f.to_ndarray(format = "rgb24") for f in container.decode(container.streams.video[0])]
|
||||
audio = None
|
||||
if container.streams.audio:
|
||||
container.close()
|
||||
container = av.open(str(path))
|
||||
chunks = [c.to_ndarray() for c in container.decode(container.streams.audio[0])]
|
||||
if chunks:
|
||||
audio = np.concatenate([c.reshape(c.shape[0], -1).mean(axis = 0) for c in chunks])
|
||||
container.close()
|
||||
return frames, audio
|
||||
|
||||
|
||||
# ── configuration plumbing ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def parse_spec(spec: str) -> dict[str, str]:
|
||||
"""'k=v;k=v' (or space-free 'k=v,k=v') -> dict; empty string -> {} (pure base)."""
|
||||
out: dict[str, str] = {}
|
||||
for part in spec.replace(",", ";").split(";"):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
if "=" not in part:
|
||||
raise ValueError(f"Bad candidate spec fragment '{part}' (expected key=value)")
|
||||
key, value = part.split("=", 1)
|
||||
out[key.strip()] = value.strip()
|
||||
return out
|
||||
|
||||
|
||||
def spec_label(spec: dict[str, str]) -> str:
|
||||
if not spec:
|
||||
return "base"
|
||||
return ",".join(f"{k}={Path(v).name if k == 'gguf_filename' else v}" for k, v in sorted(spec.items()))
|
||||
|
||||
|
||||
def run_config(backend: Any, args: Any, spec: dict[str, str], workdir: Path, name: str) -> dict[str, Any]:
|
||||
"""Load per spec, generate the fixed clip, unload. Returns frames/audio/cost."""
|
||||
import torch
|
||||
|
||||
load_kwargs: dict[str, Any] = {
|
||||
"gguf_filename": spec.get("gguf_filename"),
|
||||
"model_kind": spec.get("model_kind", args.model_kind),
|
||||
"memory_mode": spec.get("memory_mode"),
|
||||
"speed_mode": spec.get("speed_mode"),
|
||||
"attention_backend": spec.get("attention_backend"),
|
||||
"transformer_cache": spec.get("transformer_cache"),
|
||||
"transformer_quant": spec.get("transformer_quant"),
|
||||
}
|
||||
t0 = time.monotonic()
|
||||
status = backend.load_pipeline(args.model, **load_kwargs)
|
||||
load_s = time.monotonic() - t0
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
t0 = time.monotonic()
|
||||
result = backend.generate(
|
||||
prompt = args.prompt,
|
||||
width = args.width,
|
||||
height = args.height,
|
||||
num_frames = args.frames,
|
||||
fps = args.fps,
|
||||
steps = args.steps,
|
||||
guidance = args.guidance,
|
||||
seed = args.seed,
|
||||
)
|
||||
generate_s = time.monotonic() - t0
|
||||
peak_gib = (
|
||||
torch.cuda.max_memory_allocated() / 2**30 if torch.cuda.is_available() else 0.0
|
||||
)
|
||||
backend.unload()
|
||||
frames, audio = decode_mp4(result["mp4_bytes"], workdir, name)
|
||||
return {
|
||||
"frames": frames,
|
||||
"audio": audio,
|
||||
"load_s": round(load_s, 1),
|
||||
"generate_s": round(generate_s, 1),
|
||||
"peak_vram_gib": round(peak_gib, 2),
|
||||
"resolved": {k: v for k, v in status.items() if k in (
|
||||
"speed_mode", "attention_backend", "transformer_cache", "transformer_quant",
|
||||
"offload_policy", "model_kind",
|
||||
)},
|
||||
}
|
||||
|
||||
|
||||
def run_gate(args: Any) -> int:
|
||||
from core.inference.video import get_video_backend
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(parents = True, exist_ok = True)
|
||||
backend = get_video_backend()
|
||||
|
||||
print(f"reference: {args.reference or 'base'}", flush = True)
|
||||
ref = run_config(backend, args, parse_spec(args.reference), out_dir, "reference")
|
||||
print(
|
||||
f" load {ref['load_s']}s, generate {ref['generate_s']}s, "
|
||||
f"peak {ref['peak_vram_gib']} GiB",
|
||||
flush = True,
|
||||
)
|
||||
|
||||
rows = []
|
||||
for spec_str in args.candidates:
|
||||
spec = parse_spec(spec_str)
|
||||
label = spec_label(spec)
|
||||
print(f"candidate: {label}", flush = True)
|
||||
cand = run_config(backend, args, spec, out_dir, label.replace("/", "_").replace("=", "-"))
|
||||
metrics = clip_metrics(ref["frames"], cand["frames"], sample_count = args.sample_frames)
|
||||
audio = audio_metrics(ref["audio"], cand["audio"])
|
||||
row = {
|
||||
"candidate": label,
|
||||
**{k: (round(v, 4) if isinstance(v, float) and math.isfinite(v) else v)
|
||||
for k, v in metrics.items()},
|
||||
**{f"audio_{k}": v for k, v in audio.items()},
|
||||
"load_s": cand["load_s"],
|
||||
"generate_s": cand["generate_s"],
|
||||
"ref_generate_s": ref["generate_s"],
|
||||
"peak_vram_gib": cand["peak_vram_gib"],
|
||||
"resolved": cand["resolved"],
|
||||
"verdict": verdict(metrics, audio),
|
||||
}
|
||||
rows.append(row)
|
||||
print(
|
||||
f" ssim {row['ssim_mean']:.3f} | psnr {row['psnr_mean']:.1f} dB | "
|
||||
f"temporal {row['temporal_deviation']:.3f} | luma>={row['min_luma']:.3f} | "
|
||||
f"gen {row['generate_s']}s (ref {ref['generate_s']}s) | "
|
||||
f"vram {row['peak_vram_gib']} GiB | {row['verdict']}",
|
||||
flush = True,
|
||||
)
|
||||
|
||||
report = {
|
||||
"model": args.model,
|
||||
"reference": args.reference or "base",
|
||||
"prompt": args.prompt,
|
||||
"shape": [args.width, args.height, args.frames, args.fps],
|
||||
"steps": args.steps,
|
||||
"guidance": args.guidance,
|
||||
"seed": args.seed,
|
||||
"reference_cost": {k: ref[k] for k in ("load_s", "generate_s", "peak_vram_gib")},
|
||||
"candidates": rows,
|
||||
}
|
||||
(out_dir / "report.json").write_text(json.dumps(report, indent = 1))
|
||||
print(f"report: {out_dir / 'report.json'}", flush = True)
|
||||
return 0 if all(r["verdict"] != "FAIL" for r in rows) else 1
|
||||
|
||||
|
||||
# ── selftest (CPU-only, synthetic clips, no torch/model) ────────────────────
|
||||
|
||||
|
||||
def selftest() -> int:
|
||||
import numpy as np
|
||||
|
||||
rng = np.random.default_rng(0)
|
||||
h, w, n = 64, 96, 12
|
||||
|
||||
def make_clip(offset = 0.0, noise = 0.0, black = False):
|
||||
frames = []
|
||||
for t in range(n):
|
||||
x = np.linspace(0, 1, w)[None, :] + t * 0.05 + offset
|
||||
base = (np.sin(x * 6.283) * 0.5 + 0.5) * 255.0
|
||||
frame = np.repeat(base[..., None], 3, axis = 2) * np.ones((h, 1, 1))
|
||||
if noise:
|
||||
frame = frame + rng.normal(0, noise, frame.shape)
|
||||
if black:
|
||||
frame = frame * 0.0
|
||||
frames.append(np.clip(frame, 0, 255).astype(np.uint8))
|
||||
return frames
|
||||
|
||||
ref = make_clip()
|
||||
ok = True
|
||||
|
||||
def check(cond, msg):
|
||||
nonlocal ok
|
||||
print(("PASS: " if cond else "FAIL: ") + msg)
|
||||
ok = ok and cond
|
||||
|
||||
same = clip_metrics(ref, make_clip())
|
||||
check(same["ssim_mean"] > 0.99 and same["temporal_deviation"] < 0.01,
|
||||
f"identical clip scores ~1 (ssim {same['ssim_mean']:.3f})")
|
||||
check(verdict(same, {"silent_collapse": False}) == "PASS", "identical clip verdict PASS")
|
||||
|
||||
noisy = clip_metrics(ref, make_clip(noise = 12.0))
|
||||
check(0.3 < noisy["ssim_mean"] < 0.99, f"noisy clip degrades ssim ({noisy['ssim_mean']:.3f})")
|
||||
|
||||
black = clip_metrics(ref, make_clip(black = True))
|
||||
check(verdict(black, {"silent_collapse": False}) == "FAIL",
|
||||
f"black clip verdict FAIL (min_luma {black['min_luma']:.3f})")
|
||||
|
||||
shifted = clip_metrics(ref, make_clip(offset = 0.5))
|
||||
check(shifted["ssim_mean"] < same["ssim_mean"], "content shift lowers ssim")
|
||||
|
||||
audio = audio_metrics(np.sin(np.linspace(0, 100, 16000)), np.zeros(16000))
|
||||
check(audio["silent_collapse"] is True, "silent audio collapse detected")
|
||||
audio_ok = audio_metrics(np.sin(np.linspace(0, 100, 16000)),
|
||||
np.sin(np.linspace(0, 100, 16000)) * 0.8)
|
||||
check(audio_ok["silent_collapse"] is False, "attenuated audio is not a collapse")
|
||||
|
||||
print("VIDEO-QUALITY-SELFTEST", "PASS" if ok else "FAIL")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description = __doc__.split("\n")[0])
|
||||
parser.add_argument("--selftest", action = "store_true", help = "CPU metric sanity check")
|
||||
parser.add_argument("--model", help = "Repo id handed to the video backend")
|
||||
parser.add_argument("--model-kind", default = None, help = "pipeline | gguf | single_file")
|
||||
parser.add_argument("--reference", default = "", help = "Reference spec 'k=v;k=v' ('' = plain base load)")
|
||||
parser.add_argument("--candidates", nargs = "+", default = [], help = "Candidate specs 'k=v;k=v'")
|
||||
parser.add_argument("--prompt", default = DEFAULT_PROMPT)
|
||||
parser.add_argument("--width", type = int, default = 768)
|
||||
parser.add_argument("--height", type = int, default = 512)
|
||||
parser.add_argument("--frames", type = int, default = 33)
|
||||
parser.add_argument("--fps", type = int, default = 24)
|
||||
parser.add_argument("--steps", type = int, default = None)
|
||||
parser.add_argument("--guidance", type = float, default = None)
|
||||
parser.add_argument("--seed", type = int, default = 7)
|
||||
parser.add_argument("--sample-frames", type = int, default = 5)
|
||||
parser.add_argument("--out-dir", default = "outputs/video_quality")
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.selftest:
|
||||
return selftest()
|
||||
if not args.model or not args.candidates:
|
||||
parser.error("--model and --candidates are required (or use --selftest)")
|
||||
return run_gate(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -30,6 +30,7 @@ family's official base repos, or a local path the user explicitly picked.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import inspect
|
||||
import os
|
||||
import tempfile
|
||||
|
|
@ -100,6 +101,11 @@ _TRUSTED_NON_GGUF_VIDEO_REPOS = frozenset(
|
|||
# remote code, so allowed as full (pipeline-kind) loads like the LTX-2 bases.
|
||||
"wan-ai/wan2.2-ti2v-5b-diffusers",
|
||||
"wan-ai/wan2.2-t2v-a14b-diffusers",
|
||||
# HunyuanVideo-1.5 community Diffusers repacks (tencent's own repo is the
|
||||
# original non-diffusers layout: config.json, no model_index.json, so it
|
||||
# cannot load through HunyuanVideo15Pipeline at all).
|
||||
"hunyuanvideo-community/hunyuanvideo-1.5-diffusers-480p_t2v",
|
||||
"hunyuanvideo-community/hunyuanvideo-1.5-diffusers-720p_t2v",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -129,6 +135,37 @@ def _is_trusted_video_repo(repo_id: str) -> bool:
|
|||
return rid.startswith("unsloth/") or rid in _TRUSTED_NON_GGUF_VIDEO_REPOS
|
||||
|
||||
|
||||
class _VideoGenerationCancelled(Exception):
|
||||
"""Unwinds a denoise loop that has no cooperative interrupt (no step callback);
|
||||
generate() maps it to the VIDEO_CANCELLED_MSG sentinel the routes 409 on."""
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _scheduler_step_progress(pipe: Any, on_step: Any):
|
||||
"""Progress + cancellation for pipelines WITHOUT callback_on_step_end.
|
||||
|
||||
HunyuanVideo15Pipeline exposes no per-step callback, but every denoise step
|
||||
makes exactly one ``scheduler.step`` call, so wrapping that method gives the
|
||||
same per-step tick the callback path gets. ``on_step`` receives the 1-based
|
||||
step count and may raise (_VideoGenerationCancelled) to abort the loop. The
|
||||
original method is always restored, even when the pipeline raises.
|
||||
"""
|
||||
scheduler = pipe.scheduler
|
||||
original = scheduler.step
|
||||
count = {"n": 0}
|
||||
|
||||
def _step(*args: Any, **kwargs: Any) -> Any:
|
||||
count["n"] += 1
|
||||
on_step(count["n"])
|
||||
return original(*args, **kwargs)
|
||||
|
||||
scheduler.step = _step
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
scheduler.step = original
|
||||
|
||||
|
||||
def _ensure_mp4_encoder_available() -> None:
|
||||
"""Fail a load fast when PyAV is missing: the export otherwise dies AFTER a
|
||||
multi-minute denoise, which is the worst possible time to learn about it."""
|
||||
|
|
@ -835,12 +872,19 @@ class VideoBackend:
|
|||
kwargs: dict[str, Any] = {
|
||||
"prompt": prompt,
|
||||
"num_inference_steps": steps,
|
||||
fam.cfg_kwarg: guidance,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"num_frames": frames,
|
||||
"generator": generator,
|
||||
}
|
||||
if fam.guidance_via_guider:
|
||||
# HunyuanVideo-1.5: __call__ has no guidance kwarg at all; the
|
||||
# CFG scale is a plain attribute on the pipeline's guider
|
||||
# component, set per request. Near-1 scales auto-disable CFG
|
||||
# inside the guider itself (_is_cfg_enabled's is_close check).
|
||||
pipe.guider.guidance_scale = float(guidance)
|
||||
else:
|
||||
kwargs[fam.cfg_kwarg] = guidance
|
||||
if negative_prompt and "negative_prompt" in call_params:
|
||||
kwargs["negative_prompt"] = negative_prompt
|
||||
# LTX-2 takes frame_rate (it shapes the audio track length); other
|
||||
|
|
@ -869,23 +913,42 @@ class VideoBackend:
|
|||
"started": started, "eta_seconds": None, "error": None,
|
||||
}
|
||||
|
||||
def _on_step(p, step_index, timestep, callback_kwargs):
|
||||
if cancel.is_set():
|
||||
p._interrupt = True
|
||||
return callback_kwargs
|
||||
done = step_index + 1
|
||||
def _tick(done: int) -> None:
|
||||
elapsed = time.monotonic() - started
|
||||
self._gen.update(
|
||||
step = done,
|
||||
eta_seconds = (elapsed / max(1, done)) * max(0, steps - done),
|
||||
)
|
||||
|
||||
def _on_step(p, step_index, timestep, callback_kwargs):
|
||||
if cancel.is_set():
|
||||
p._interrupt = True
|
||||
return callback_kwargs
|
||||
_tick(step_index + 1)
|
||||
return callback_kwargs
|
||||
|
||||
def _on_scheduler_step(done: int) -> None:
|
||||
# No cooperative _interrupt here: without a callback the pipeline
|
||||
# never checks it, so cancellation must unwind the denoise loop
|
||||
# via an exception (mapped to the cancelled sentinel below).
|
||||
if cancel.is_set():
|
||||
raise _VideoGenerationCancelled()
|
||||
_tick(done)
|
||||
|
||||
if "callback_on_step_end" in call_params:
|
||||
kwargs["callback_on_step_end"] = _on_step
|
||||
progress_ctx = contextlib.nullcontext()
|
||||
else:
|
||||
# HunyuanVideo-1.5 has no step callback; every scheduler.step
|
||||
# call is exactly one denoise step, so wrap it for progress +
|
||||
# cancel and restore it afterwards.
|
||||
progress_ctx = _scheduler_step_progress(pipe, _on_scheduler_step)
|
||||
|
||||
with torch.inference_mode():
|
||||
output = pipe(**kwargs)
|
||||
try:
|
||||
with torch.inference_mode(), progress_ctx:
|
||||
output = pipe(**kwargs)
|
||||
except _VideoGenerationCancelled:
|
||||
raise RuntimeError(VIDEO_CANCELLED_MSG) from None
|
||||
if cancel.is_set():
|
||||
raise RuntimeError(VIDEO_CANCELLED_MSG)
|
||||
|
||||
|
|
|
|||
|
|
@ -50,6 +50,12 @@ class VideoFamily:
|
|||
transformer2_class: Optional[str] = None
|
||||
is_moe: bool = False
|
||||
cfg2_kwarg: Optional[str] = None
|
||||
# HunyuanVideo-1.5 style guidance: the pipeline __call__ takes NO guidance
|
||||
# kwarg at all; CFG lives on a ``guider`` component (ClassifierFreeGuidance)
|
||||
# whose ``guidance_scale`` is a plain attribute set per request. When True,
|
||||
# generate() writes the scale onto ``pipe.guider`` instead of passing
|
||||
# ``cfg_kwarg`` (which the pipeline would reject as an unexpected argument).
|
||||
guidance_via_guider: bool = False
|
||||
# Generation defaults + shape constraints. ``frame_step`` is the temporal
|
||||
# compression: a valid frame count is k * frame_step + 1 (the +1 is the
|
||||
# anchor frame), so requests are snapped BEFORE latents are allocated.
|
||||
|
|
@ -190,6 +196,46 @@ _FAMILIES: tuple[VideoFamily, ...] = (
|
|||
bf16_components_gb = (114.3, 11.4, 0.5),
|
||||
gguf_repo = "QuantStack/Wan2.2-T2V-A14B-GGUF",
|
||||
),
|
||||
# HunyuanVideo-1.5 (diffusers >= 0.39): an 8.3B video DiT with a Qwen2.5-VL
|
||||
# text encoder plus a ByT5 glyph encoder. Three quirks, all verified against
|
||||
# the installed pipeline source (pipeline_hunyuan_video1_5.py):
|
||||
# 1. __call__ takes NO guidance kwarg; CFG lives on the ``guider`` component
|
||||
# (ClassifierFreeGuidance; the 480p t2v repo ships guidance_scale = 6.0),
|
||||
# hence guidance_via_guider.
|
||||
# 2. __call__ has NO callback_on_step_end; generate() falls back to the
|
||||
# scheduler.step progress wrapper automatically (capability-detected).
|
||||
# 3. The tencent/HunyuanVideo-1.5 repo is the ORIGINAL layout (config.json,
|
||||
# no model_index.json); only the hunyuanvideo-community Diffusers repacks
|
||||
# load through HunyuanVideo15Pipeline, so those are the trusted repos.
|
||||
# The transformer declares _repeated_blocks and inherits CacheMixin, so the
|
||||
# regional compile profile and First-Block-Cache both apply.
|
||||
VideoFamily(
|
||||
name = "hunyuanvideo-1.5",
|
||||
pipeline_class = "HunyuanVideo15Pipeline",
|
||||
transformer_class = "HunyuanVideo15Transformer3DModel",
|
||||
base_repo = "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
||||
# No bare "hunyuanvideo" alias: it would also claim the incompatible 1.0
|
||||
# repos (HunyuanVideoPipeline), which this family cannot load.
|
||||
aliases = ("hunyuanvideo-1-5", "hunyuanvideo1.5", "hunyuanvideo1-5", "hv15"),
|
||||
has_audio = False,
|
||||
guidance_via_guider = True,
|
||||
default_steps = 50,
|
||||
default_guidance = 6.0,
|
||||
# 121 frames at 24 fps is ~5s, the pipeline's own num_frames default.
|
||||
default_num_frames = 121,
|
||||
default_fps = 24,
|
||||
# The HV15 VAE compresses 16x spatial / 4x temporal (vae config:
|
||||
# spatial_compression_ratio 16, temporal_compression_ratio 4) with a
|
||||
# patch-1 transformer, so sizes snap to /16 and frames to 4k+1.
|
||||
frame_step = 4,
|
||||
resolution_multiple = 16,
|
||||
# 480p-class presets (the base repo is the 480p t2v variant): landscape,
|
||||
# vertical, square.
|
||||
resolution_presets = ((832, 480), (480, 832), (624, 624)),
|
||||
# Disk shards are fp32 for the DiT (32.0 GB -> 16.6 bf16-resident) and the
|
||||
# VAE (4.7 -> 2.4); the Qwen2.5-VL TE is stored bf16 (14.0) plus ByT5 0.8.
|
||||
bf16_components_gb = (16.6, 14.8, 2.4),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -262,6 +308,9 @@ _VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
|
|||
# and the base repo. A future distilled Wan GGUF is caught by the "distilled"
|
||||
# row above (listed first), exactly as the LTX-2.3 distilled checkpoints are.
|
||||
("wan", 50, 5.0),
|
||||
# HunyuanVideo-1.5 runs the pipeline's 50 steps with the guider's shipped
|
||||
# CFG 6.0 (guider_config.json in the community Diffusers repacks).
|
||||
("hunyuanvideo", 50, 6.0),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import types
|
|||
import pytest
|
||||
|
||||
from core.inference.video import VideoBackend, get_video_backend, resolve_video_model_kind
|
||||
from core.inference.video_families import VIDEO_NOT_LOADED_MSG
|
||||
from core.inference.video_families import VIDEO_CANCELLED_MSG, VIDEO_NOT_LOADED_MSG
|
||||
|
||||
|
||||
class _FakeDtype:
|
||||
|
|
@ -273,6 +273,82 @@ class _FakeWanPipelineSingle:
|
|||
return _FakeWanPipeMoE() if moe else _FakeWanPipeSingle()
|
||||
|
||||
|
||||
# ── HunyuanVideo-1.5 fakes: the __call__ signature has NO guidance kwarg and NO
|
||||
# callback_on_step_end (matching pipeline_hunyuan_video1_5.py in diffusers 0.39),
|
||||
# a guider object carries the CFG scale, and the denoise loop drives
|
||||
# scheduler.step -- so the guider write and the scheduler-wrap progress/cancel
|
||||
# paths actually exercise.
|
||||
|
||||
|
||||
class _FakeHV15Scheduler:
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
# Test hook fired from the ORIGINAL step (i.e. inside the wrapped call),
|
||||
# letting a test cancel mid-denoise exactly as a user request would land.
|
||||
self.on_step = None
|
||||
|
||||
def step(self, *args, **kwargs):
|
||||
self.calls += 1
|
||||
if self.on_step is not None:
|
||||
self.on_step(self.calls)
|
||||
return object()
|
||||
|
||||
|
||||
class _FakeHV15Pipe:
|
||||
def __init__(self) -> None:
|
||||
self.vae = _FakeWanVae()
|
||||
self.transformer = _FakeWanDiT()
|
||||
self.scheduler = _FakeHV15Scheduler()
|
||||
self.guider = types.SimpleNamespace(guidance_scale = 6.0)
|
||||
self.components = {"transformer": self.transformer, "vae": self.vae}
|
||||
self.moved_to = None
|
||||
self.last_kwargs = None
|
||||
|
||||
def to(self, device):
|
||||
self.moved_to = device
|
||||
return self
|
||||
|
||||
def enable_vae_tiling(self) -> None:
|
||||
self.vae.tiled = True
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
prompt = None,
|
||||
negative_prompt = None,
|
||||
height = None,
|
||||
width = None,
|
||||
num_frames = None,
|
||||
num_inference_steps = None,
|
||||
generator = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.last_kwargs = {
|
||||
"prompt": prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
"num_inference_steps": num_inference_steps,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"num_frames": num_frames,
|
||||
**kwargs,
|
||||
}
|
||||
for _ in range(int(num_inference_steps or 1)):
|
||||
self.scheduler.step()
|
||||
frames = [[object() for _ in range(int(num_frames or 1))]]
|
||||
return types.SimpleNamespace(frames = frames, audio = None)
|
||||
|
||||
|
||||
class _FakeHV15Pipeline:
|
||||
last: dict = {}
|
||||
instance = None
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, repo, **kwargs):
|
||||
_FakeHV15Pipeline.last = {"repo": repo, **kwargs}
|
||||
_FakeHV15Pipeline.instance = _FakeHV15Pipe()
|
||||
return _FakeHV15Pipeline.instance
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_runtime(monkeypatch):
|
||||
torch = types.ModuleType("torch")
|
||||
|
|
@ -291,6 +367,8 @@ def fake_runtime(monkeypatch):
|
|||
# Wan2.2: one pipeline class serves both families (it dispatches on the repo id).
|
||||
diffusers.WanPipeline = _FakeWanPipelineSingle
|
||||
diffusers.WanTransformer3DModel = _FakeTransformer
|
||||
diffusers.HunyuanVideo15Pipeline = _FakeHV15Pipeline
|
||||
diffusers.HunyuanVideo15Transformer3DModel = _FakeTransformer
|
||||
diffusers.FirstBlockCacheConfig = lambda threshold = None: ("fbcache", threshold)
|
||||
|
||||
monkeypatch.setitem(sys.modules, "torch", torch)
|
||||
|
|
@ -500,6 +578,50 @@ def test_generate_progress_and_cancel_idle(fake_runtime):
|
|||
assert backend.cancel_generate() is False
|
||||
|
||||
|
||||
def test_hv15_guider_and_scheduler_progress(fake_runtime):
|
||||
# HunyuanVideo-1.5: no guidance kwarg (CFG set on the guider), no step
|
||||
# callback (progress via the scheduler.step wrapper, restored afterwards).
|
||||
backend = VideoBackend()
|
||||
status = backend.load_pipeline(
|
||||
"hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
||||
model_kind = "pipeline",
|
||||
)
|
||||
assert status["family"] == "hunyuanvideo-1.5"
|
||||
assert status["has_audio"] is False
|
||||
assert status["defaults"]["frame_step"] == 4
|
||||
|
||||
pipe = _FakeHV15Pipeline.instance
|
||||
result = backend.generate(
|
||||
prompt = "a fox in the snow", steps = 4, guidance = 3.5, num_frames = 9, fps = 24
|
||||
)
|
||||
assert "guidance_scale" not in pipe.last_kwargs
|
||||
assert "callback_on_step_end" not in pipe.last_kwargs
|
||||
assert pipe.guider.guidance_scale == 3.5
|
||||
# One wrapped tick per denoise step, then the original method back in place.
|
||||
assert pipe.scheduler.calls == 4
|
||||
assert pipe.scheduler.step.__func__ is _FakeHV15Scheduler.step
|
||||
assert result["num_frames"] == 9 and result["has_audio"] is False
|
||||
|
||||
|
||||
def test_hv15_cancel_unwinds_scheduler_loop(fake_runtime):
|
||||
backend = VideoBackend()
|
||||
backend.load_pipeline(
|
||||
"hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
||||
model_kind = "pipeline",
|
||||
)
|
||||
pipe = _FakeHV15Pipeline.instance
|
||||
# Cancel lands during the FIRST real step; the next wrapped call must raise out
|
||||
# of the denoise loop and generate() must surface the cancelled sentinel.
|
||||
pipe.scheduler.on_step = (
|
||||
lambda n: backend.cancel_generate() if n == 1 else None
|
||||
)
|
||||
with pytest.raises(RuntimeError, match = VIDEO_CANCELLED_MSG):
|
||||
backend.generate(prompt = "a fox", steps = 4)
|
||||
assert pipe.scheduler.calls == 1
|
||||
# The wrapper must restore scheduler.step even on the exception path.
|
||||
assert pipe.scheduler.step.__func__ is _FakeHV15Scheduler.step
|
||||
|
||||
|
||||
def test_singleton():
|
||||
assert get_video_backend() is get_video_backend()
|
||||
|
||||
|
|
|
|||
|
|
@ -150,6 +150,7 @@ def test_supported_names():
|
|||
"ltx-2",
|
||||
"wan2.2-ti2v-5b",
|
||||
"wan2.2-t2v-a14b",
|
||||
"hunyuanvideo-1.5",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -200,3 +201,25 @@ def test_family_size_table_present():
|
|||
# that would let auto planning under-reserve by ~50 GB.
|
||||
assert text_encoder_gb > transformer_gb > 20.0
|
||||
assert companions_gb > 0.0
|
||||
|
||||
|
||||
def test_hv15_detection_and_flags():
|
||||
fam = detect_video_family("hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v")
|
||||
assert fam is not None and fam.name == "hunyuanvideo-1.5"
|
||||
# CFG lives on the guider component (no guidance kwarg in __call__), and the
|
||||
# HV15 VAE compresses 16x spatial / 4x temporal.
|
||||
assert fam.guidance_via_guider is True
|
||||
assert fam.frame_step == 4 and fam.resolution_multiple == 16
|
||||
assert fam.has_audio is False
|
||||
assert detect_video_family("x/y", override = "hv15") is fam
|
||||
# The incompatible HunyuanVideo 1.0 repos must NOT be claimed: their
|
||||
# model_index pins HunyuanVideoPipeline, which this family cannot load.
|
||||
assert detect_video_family("hunyuanvideo-community/HunyuanVideo") is None
|
||||
|
||||
|
||||
def test_hv15_generation_defaults():
|
||||
# The community repacks ship a guider with guidance_scale 6.0 and the
|
||||
# pipeline's own 50-step schedule.
|
||||
assert default_video_generation_params(
|
||||
None, "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v"
|
||||
) == (50, 6.0)
|
||||
|
|
|
|||
|
|
@ -72,6 +72,9 @@ const PIPELINE_MODELS: Record<string, PipelineSpec> = {
|
|||
// these to the Wan-AI base repos (see _TRUSTED_NON_GGUF_VIDEO_REPOS).
|
||||
"Wan-AI/Wan2.2-TI2V-5B-Diffusers": { kind: "pipeline" },
|
||||
"Wan-AI/Wan2.2-T2V-A14B-Diffusers": { kind: "pipeline" },
|
||||
// HunyuanVideo-1.5 community Diffusers repack (tencent's own repo is the original
|
||||
// non-diffusers layout and cannot load as a pipeline).
|
||||
"hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v": { kind: "pipeline" },
|
||||
};
|
||||
|
||||
// A curated GGUF picker entry: isGguf true expands its .gguf files in the quant expander
|
||||
|
|
@ -106,6 +109,11 @@ const VIDEO_MODELS: ModelOption[] = [
|
|||
"Wan 2.2 T2V A14B (MoE)",
|
||||
"Text-to-video, dual-expert · Safetensors",
|
||||
),
|
||||
pipelineModel(
|
||||
"hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v",
|
||||
"HunyuanVideo 1.5 (480p)",
|
||||
"Text-to-video 480p · Safetensors",
|
||||
),
|
||||
];
|
||||
|
||||
// Per-model generation defaults (steps + guidance), matched by repo-id substring, most
|
||||
|
|
@ -120,6 +128,9 @@ const MODEL_DEFAULTS: Array<{ match: string; steps: number; guidance: number }>
|
|||
// Wan2.2 pipelines default to 50 steps at CFG 5.0 (WanPipeline defaults, verified in
|
||||
// diffusers 0.39). The backend supplies the fps per family (24 for TI2V-5B, 16 for A14B).
|
||||
{ match: "wan", steps: 50, guidance: 5 },
|
||||
// HunyuanVideo-1.5 runs 50 steps; guidance 6 matches the guider the repo ships
|
||||
// (the backend writes it onto the guider component, there is no pipeline kwarg).
|
||||
{ match: "hunyuanvideo", steps: 50, guidance: 6 },
|
||||
];
|
||||
|
||||
function defaultsFor(repoId: string): { steps: number; guidance: number } {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue