Add Wan2.2-I2V-A14B image-to-video support to the video backend
Wan-AI/Wan2.2-I2V-A14B-Diffusers is the image-to-video sibling of the already supported T2V-A14B: the same dual-expert WanTransformer3DModel pair (boundary_ratio 0.9 in the pipeline config) behind WanImageToVideoPipeline, conditioning through the VAE latent (no CLIP-vision image encoder in this 2.2 variant). - New wan2.2-i2v-a14b family: image_conditioned flag, card recipe defaults (40 steps, CFG 3.5, 81 frames at 16 fps), the T2V memory table (57.2 GB both experts bf16), fp32-pinned VAE, and a wan2.2-i2v generation-defaults key ahead of the generic wan 50/5.0 entry. - Source-image plumbing: /video/generate takes init_image (base64/data URL); begin_generate 400s synchronously when an image-to-video family has no image or a text-only family is given one, and generate() decodes and resizes the image to the snapped output size before threading it as the pipeline's image kwarg. status() reports image_input so the UI can gate the control. - Trust the official -Diffusers repo for pipeline loads and transfer the measured wan quant recipes: fp8 keeps condition_embedder in bf16 (zero padding-row scale), mxfp8 and nvfp4 stay denied, the UMT5 auto TE quant resolves dense, and the balanced FBCache pin carries over. All tables share the T2V DiT profile. - Tests: family detection/aliases/defaults, the dual-DiT image pipeline load, the image gates on both begin_generate and generate, init_image route pass-through, and the quant exclude/deny/auto entries. GPU-verified on a B200: bf16 resident load (offload none, 72.8 GB peak) animating a conditioning image at 832x480/33f in 28.7 s with first-frame MAE 5.7 vs the source, and an int8 load with both experts quantized (43.1 GB peak, clean output).
This commit is contained in:
Daniel Han
parent
63cbf7878e
commit
cffee73135
11 changed files with 281 additions and 2 deletions
|
|
@ -89,6 +89,8 @@ _FBCACHE_QUALITY_THRESHOLDS: dict[str, tuple[float, float]] = {
|
|||
# 0.08 is the least-drift cache-on point, not a compliant one.
|
||||
_FAMILY_FBCACHE_THRESHOLDS: dict[tuple[str, str], tuple[float, float]] = {
|
||||
("wan2.2-t2v-a14b", CQ_BALANCED): (DEFAULT_FBCACHE_THRESHOLD, DEFAULT_FBCACHE_THRESHOLD),
|
||||
# I2V-A14B shares the T2V DiT pair; inherit the balanced pin (same drift profile).
|
||||
("wan2.2-i2v-a14b", CQ_BALANCED): (DEFAULT_FBCACHE_THRESHOLD, DEFAULT_FBCACHE_THRESHOLD),
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -131,7 +131,8 @@ _TE_FAMILY_SCHEME_DENY: dict[str, frozenset[str]] = {
|
|||
# amplifies the perturbation ~3x harder than on wan2.2-ti2v-5b (0.0396, kept quantized there for
|
||||
# a real 1.09x on its faster DiT).
|
||||
_TE_AUTO_DENSE_FAMILIES: frozenset[str] = frozenset(
|
||||
{"hunyuanvideo-1.5", "hunyuanvideo-1.5-720p", "wan2.2-t2v-a14b"}
|
||||
# wan2.2-i2v-a14b inherits the T2V entry: same UMT5 encoder and MoE trajectory.
|
||||
{"hunyuanvideo-1.5", "hunyuanvideo-1.5-720p", "wan2.2-t2v-a14b", "wan2.2-i2v-a14b"}
|
||||
)
|
||||
|
||||
# Map a TE torchao scheme to the transformer smoke-probe scheme (same GEMM), so ``auto`` degrades
|
||||
|
|
|
|||
|
|
@ -127,6 +127,7 @@ _INT8_FAMILY_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = {
|
|||
_FP8_FAMILY_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = {
|
||||
"wan2.2-ti2v-5b": ("condition_embedder",),
|
||||
"wan2.2-t2v-a14b": ("condition_embedder",), # same DiT class + padded-text conditioning
|
||||
"wan2.2-i2v-a14b": ("condition_embedder",), # same DiT class + padded-text conditioning
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -195,6 +196,7 @@ _FAMILY_SCHEME_DENY: dict[str, frozenset[str]] = {
|
|||
# (same scaled_mm family, not separately validated). 5B TI2V and A14B share the DiT + profile.
|
||||
"wan2.2-ti2v-5b": frozenset({TQ_MXFP8, TQ_NVFP4}),
|
||||
"wan2.2-t2v-a14b": frozenset({TQ_MXFP8, TQ_NVFP4}),
|
||||
"wan2.2-i2v-a14b": frozenset({TQ_MXFP8, TQ_NVFP4}), # same DiT pair + quant profile as T2V
|
||||
# HunyuanVideo-1.5 DiT: fp8 renders black frames (LPIPS 0.82) and -- unlike Wan -- the failure
|
||||
# is NOT confinable to an input embedder: its MMDiT masks padding text tokens to zero inside
|
||||
# every block, so the context stream regenerates zero rows layer after layer (fp8 on only the
|
||||
|
|
|
|||
|
|
@ -128,6 +128,7 @@ _TRUSTED_NON_GGUF_VIDEO_REPOS = frozenset(
|
|||
# Wan2.2 official diffusers base repos: safetensors-only, no remote code.
|
||||
"wan-ai/wan2.2-ti2v-5b-diffusers",
|
||||
"wan-ai/wan2.2-t2v-a14b-diffusers",
|
||||
"wan-ai/wan2.2-i2v-a14b-diffusers",
|
||||
# HunyuanVideo-1.5 community Diffusers repacks (tencent's own repo is the
|
||||
# non-diffusers layout with no model_index.json, unloadable here).
|
||||
"hunyuanvideo-community/hunyuanvideo-1.5-diffusers-480p_t2v",
|
||||
|
|
@ -1916,6 +1917,7 @@ class VideoBackend:
|
|||
guidance: Optional[float] = None,
|
||||
guidance_2: Optional[float] = None,
|
||||
seed: Optional[int] = None,
|
||||
init_image: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Validate cheaply, then run generate + gallery persist on a daemon thread.
|
||||
|
||||
|
|
@ -1934,6 +1936,21 @@ class VideoBackend:
|
|||
raise RuntimeError(VIDEO_NOT_LOADED_MSG)
|
||||
if self._generate_job_active:
|
||||
raise RuntimeError(VIDEO_GENERATION_BUSY_MSG)
|
||||
# Image gate up front (cheap, no decode) so the POST 400s synchronously instead
|
||||
# of reporting a failed job: an image-to-video family needs a source image, a
|
||||
# text-only family has no ``image`` kwarg to take one. generate() re-checks for
|
||||
# direct callers. getattr-guarded: tests fake _state without a family.
|
||||
fam = getattr(self._state, "family", None)
|
||||
if fam is not None:
|
||||
if getattr(fam, "image_conditioned", False) and not (init_image or "").strip():
|
||||
raise ValueError(
|
||||
f"{fam.name} is an image-to-video model: attach a source image "
|
||||
"to animate."
|
||||
)
|
||||
if init_image and not getattr(fam, "image_conditioned", False):
|
||||
raise ValueError(
|
||||
f"{fam.name} is a text-to-video model and does not take a source image."
|
||||
)
|
||||
# A background compile prewarm may hold _generate_lock. Signal its dedicated
|
||||
# cancel handle BEFORE registering ours so the real job preempts the warmup at
|
||||
# its next step boundary instead of queueing behind it (and so unload/cancel
|
||||
|
|
@ -1963,6 +1980,7 @@ class VideoBackend:
|
|||
guidance = guidance,
|
||||
guidance_2 = guidance_2,
|
||||
seed = seed,
|
||||
init_image = init_image,
|
||||
cancel_event = cancel,
|
||||
),
|
||||
daemon = True,
|
||||
|
|
@ -2074,6 +2092,7 @@ class VideoBackend:
|
|||
guidance: Optional[float] = None,
|
||||
guidance_2: Optional[float] = None,
|
||||
seed: Optional[int] = None,
|
||||
init_image: Optional[str] = None,
|
||||
cancel_event: Optional[threading.Event] = None,
|
||||
) -> dict[str, Any]:
|
||||
import torch
|
||||
|
|
@ -2125,6 +2144,29 @@ class VideoBackend:
|
|||
"num_frames": frames,
|
||||
"generator": generator,
|
||||
}
|
||||
# Image-conditioned families (WanImageToVideoPipeline) REQUIRE a source image;
|
||||
# text-only families have no ``image`` kwarg to feed one to. Both mismatches are
|
||||
# client input -> ValueError (the route/worker map it to a 400-style message).
|
||||
if fam.image_conditioned:
|
||||
if not (init_image or "").strip():
|
||||
raise ValueError(
|
||||
f"{fam.name} is an image-to-video model: attach a source image "
|
||||
"to animate."
|
||||
)
|
||||
from .diffusion import _decode_b64_image
|
||||
from PIL import Image
|
||||
|
||||
init_pil = _decode_b64_image(init_image, mode = "RGB")
|
||||
# The pipeline derives the latent grid from height/width and encodes the
|
||||
# image at that size; resize here so the conditioning frame matches the
|
||||
# snapped output size exactly (no center-crop surprises).
|
||||
if init_pil.size != (width, height):
|
||||
init_pil = init_pil.resize((width, height), Image.LANCZOS)
|
||||
kwargs["image"] = init_pil
|
||||
elif init_image:
|
||||
raise ValueError(
|
||||
f"{fam.name} is a text-to-video model and does not take a source image."
|
||||
)
|
||||
if fam.guidance_via_guider:
|
||||
# HunyuanVideo-1.5: __call__ has no guidance kwarg; CFG scale is a guider
|
||||
# attribute set per request (near-1 scales auto-disable CFG in the guider).
|
||||
|
|
@ -2467,6 +2509,7 @@ class VideoBackend:
|
|||
"vae_quant": None,
|
||||
"cfg_parallel": None,
|
||||
"has_audio": False,
|
||||
"image_input": False,
|
||||
"defaults": None,
|
||||
"resolved": None,
|
||||
}
|
||||
|
|
@ -2497,6 +2540,9 @@ class VideoBackend:
|
|||
"vae_quant": state.vae_quant,
|
||||
"cfg_parallel": state.cfg_parallel,
|
||||
"has_audio": fam.has_audio,
|
||||
# True for image-to-video families: the UI shows the source-image control and
|
||||
# requires an image before submitting.
|
||||
"image_input": fam.image_conditioned,
|
||||
"defaults": {
|
||||
"steps": default_steps,
|
||||
"guidance": default_guidance,
|
||||
|
|
|
|||
|
|
@ -79,6 +79,10 @@ class VideoFamily:
|
|||
vae_force_fp32: bool = False
|
||||
# Curated GGUF repo for the picker (the DiT as single-file GGUF quants).
|
||||
gguf_repo: Optional[str] = None
|
||||
# True when the pipeline REQUIRES a conditioning image (WanImageToVideoPipeline): the
|
||||
# generate path decodes/resizes the request's init_image and refuses a run without one;
|
||||
# the UI shows the source-image control only for these families.
|
||||
image_conditioned: bool = False
|
||||
|
||||
|
||||
_FAMILIES: tuple[VideoFamily, ...] = (
|
||||
|
|
@ -167,6 +171,38 @@ _FAMILIES: tuple[VideoFamily, ...] = (
|
|||
vae_force_fp32 = True,
|
||||
# No gguf_repo: community GGUFs split the experts, and a single-file load covers only one.
|
||||
),
|
||||
# Wan2.2-I2V-A14B (diffusers >= 0.35, verified on 0.39): the image-to-video dual-expert MoE.
|
||||
# Same DiT pair as T2V-A14B (boundary_ratio 0.9 vs T2V's 0.875, read from the pipeline
|
||||
# config -- no plumbing), but the pipeline is WanImageToVideoPipeline and REQUIRES a
|
||||
# conditioning image: this Wan2.2 variant conditions through the VAE latent (the repo's
|
||||
# image_encoder/image_processor slots are null; no CLIP-vision like Wan2.1), so generate()
|
||||
# threads the decoded init_image straight into ``image``. Card recipe: 40 steps, CFG 3.5,
|
||||
# 81 frames at 16 fps.
|
||||
VideoFamily(
|
||||
name = "wan2.2-i2v-a14b",
|
||||
pipeline_class = "WanImageToVideoPipeline",
|
||||
transformer_class = "WanTransformer3DModel",
|
||||
base_repo = "Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
aliases = ("wan2.2-i2v", "wan-i2v", "wan-i2v-a14b"),
|
||||
has_audio = False,
|
||||
transformer2_class = "WanTransformer3DModel",
|
||||
is_moe = True,
|
||||
cfg2_kwarg = "guidance_scale_2",
|
||||
image_conditioned = True,
|
||||
# The I2V card runs 40 steps at CFG 3.5 (vs the T2V/pipeline default 50/5.0).
|
||||
default_steps = 40,
|
||||
default_guidance = 3.5,
|
||||
default_num_frames = 81,
|
||||
default_fps = 16,
|
||||
frame_step = 4,
|
||||
resolution_multiple = 16,
|
||||
resolution_presets = ((1280, 720), (832, 480), (480, 832), (720, 1280)),
|
||||
# Same shipped layout as T2V-A14B: each expert fp32 on disk (57.15 GB) -> ~28.6 bf16
|
||||
# each -> ~57.2 for both; UMT5 TE bf16 (11.4); VAE fp32 (0.5).
|
||||
bf16_components_gb = (57.2, 11.4, 0.5),
|
||||
vae_force_fp32 = True,
|
||||
# No gguf_repo: community I2V GGUFs split the experts, and a single-file load covers only one.
|
||||
),
|
||||
# HunyuanVideo-1.5 (diffusers >= 0.39): 8.3B DiT, Qwen2.5-VL text encoder + ByT5 glyph
|
||||
# encoder. Three quirks: (1) __call__ has NO guidance kwarg; CFG on the ``guider``
|
||||
# (guidance_via_guider); (2) NO callback_on_step_end (generate() uses the scheduler.step
|
||||
|
|
@ -279,7 +315,9 @@ def snap_video_size(fam: VideoFamily, width: int, height: int) -> tuple[int, int
|
|||
_VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = (
|
||||
("distilled", 8, 1.0),
|
||||
("ltx", 40, 4.0),
|
||||
# Wan2.2 pipelines default to 50 steps / CFG 5.0; both TI2V-5B and A14B share these.
|
||||
# Wan2.2 I2V runs its card recipe (40 steps, CFG 3.5); must precede the generic "wan" key.
|
||||
("wan2.2-i2v", 40, 3.5),
|
||||
# Wan2.2 T2V pipelines default to 50 steps / CFG 5.0; both TI2V-5B and T2V-A14B share these.
|
||||
("wan", 50, 5.0),
|
||||
# HunyuanVideo-1.5: 50 steps with the guider's shipped CFG 6.0.
|
||||
("hunyuanvideo", 50, 6.0),
|
||||
|
|
|
|||
|
|
@ -2552,6 +2552,13 @@ class VideoGenerateRequest(BaseModel):
|
|||
seed: Optional[int] = Field(
|
||||
None, ge = 0, le = 2**53 - 1, description = "Seed for reproducibility (random if omitted)"
|
||||
)
|
||||
# Image-to-video conditioning (base64 or data-URL). Required by image-conditioned
|
||||
# families (Wan2.2-I2V); rejected with a 400 by text-only families.
|
||||
init_image: Optional[str] = Field(
|
||||
None,
|
||||
description = "Source image to animate (base64/data-URL). Required for image-to-video "
|
||||
"families; not accepted by text-to-video families.",
|
||||
)
|
||||
|
||||
|
||||
class GalleryVideo(BaseModel):
|
||||
|
|
@ -2706,6 +2713,11 @@ class VideoStatusResponse(BaseModel):
|
|||
has_audio: bool = Field(
|
||||
False, description = "Whether the loaded family produces a synchronized audio track"
|
||||
)
|
||||
image_input: bool = Field(
|
||||
False,
|
||||
description = "Whether the loaded family requires a source image (image-to-video): the "
|
||||
"UI shows the source-image control and generate refuses to run without one",
|
||||
)
|
||||
defaults: Optional[VideoGenerationDefaults] = Field(
|
||||
None, description = "Per-family generation defaults + shape constraints; null when unloaded"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -188,6 +188,7 @@ async def generate_video(
|
|||
guidance = request.guidance,
|
||||
guidance_2 = request.guidance_2,
|
||||
seed = request.seed,
|
||||
init_image = request.init_image,
|
||||
)
|
||||
except ValueError as exc:
|
||||
# Bad client input -- a 400 with the reason, not a generic 500.
|
||||
|
|
|
|||
|
|
@ -453,6 +453,8 @@ def test_exclude_tokens_for_scheme_family():
|
|||
|
||||
assert exclude_tokens_for_scheme(TQ_FP8, "wan2.2-ti2v-5b") == ("condition_embedder",)
|
||||
assert exclude_tokens_for_scheme(TQ_FP8, "wan2.2-t2v-a14b") == ("condition_embedder",)
|
||||
# I2V-A14B shares the T2V DiT pair, so the fp8 recipe transfers unchanged.
|
||||
assert exclude_tokens_for_scheme(TQ_FP8, "wan2.2-i2v-a14b") == ("condition_embedder",)
|
||||
assert exclude_tokens_for_scheme(TQ_MXFP8, "wan2.2-ti2v-5b") == ("condition_embedder",)
|
||||
# Hunyuan is not localisable (fp8 stays denied), and an unknown family gets nothing.
|
||||
assert exclude_tokens_for_scheme(TQ_FP8, "hunyuanvideo-1.5") == ()
|
||||
|
|
@ -600,6 +602,7 @@ def test_family_allows_fp8_for_wan(monkeypatch):
|
|||
_allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8})
|
||||
assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-ti2v-5b") == TQ_FP8
|
||||
assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-t2v-a14b") == TQ_FP8
|
||||
assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-i2v-a14b") == TQ_FP8
|
||||
_allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_INT8}) # fp8 unavailable -> denied mx/nvfp4 skipped
|
||||
assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-ti2v-5b") == TQ_INT8
|
||||
|
||||
|
|
|
|||
|
|
@ -276,6 +276,56 @@ class _FakeWanPipeMoE(_FakeWanPipeBase):
|
|||
return self._finish(num_inference_steps, num_frames, callback_on_step_end)
|
||||
|
||||
|
||||
class _FakeWanI2VPipe(_FakeWanPipeBase):
|
||||
"""Dual-DiT image-to-video Wan pipeline (I2V-A14B): ``image`` IS in the signature
|
||||
(WanImageToVideoPipeline.__call__ in diffusers 0.39) alongside guidance_scale_2."""
|
||||
|
||||
moe = True
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
*,
|
||||
image = None,
|
||||
prompt = None,
|
||||
negative_prompt = None,
|
||||
num_inference_steps = None,
|
||||
guidance_scale = None,
|
||||
guidance_scale_2 = None,
|
||||
width = None,
|
||||
height = None,
|
||||
num_frames = None,
|
||||
generator = None,
|
||||
callback_on_step_end = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.last_kwargs = {
|
||||
"image": image,
|
||||
"prompt": prompt,
|
||||
"negative_prompt": negative_prompt,
|
||||
"num_inference_steps": num_inference_steps,
|
||||
"guidance_scale": guidance_scale,
|
||||
"guidance_scale_2": guidance_scale_2,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"num_frames": num_frames,
|
||||
**kwargs,
|
||||
}
|
||||
with self.transformer.cache_context("cond"): # real Wan pipeline wraps the denoise loop
|
||||
pass
|
||||
return self._finish(num_inference_steps, num_frames, callback_on_step_end)
|
||||
|
||||
|
||||
class _FakeWanImageToVideoPipeline:
|
||||
"""WanImageToVideoPipeline fake (from_pretrained) for the I2V-A14B family."""
|
||||
|
||||
last: dict = {}
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, repo, **kwargs):
|
||||
_FakeWanImageToVideoPipeline.last = {"repo": repo, **kwargs}
|
||||
return _FakeWanI2VPipe()
|
||||
|
||||
|
||||
class _FakeWanPipelineSingle:
|
||||
"""WanPipeline fake (from_pretrained). One class serves both families and picks the
|
||||
single-DiT / dual-DiT pipe by the repo id, exactly as diffusers dispatches on the
|
||||
|
|
@ -387,6 +437,7 @@ def fake_runtime(monkeypatch):
|
|||
diffusers.LTX2VideoTransformer3DModel = _FakeTransformer
|
||||
# Wan2.2: one pipeline class serves both families (it dispatches on the repo id).
|
||||
diffusers.WanPipeline = _FakeWanPipelineSingle
|
||||
diffusers.WanImageToVideoPipeline = _FakeWanImageToVideoPipeline
|
||||
diffusers.WanTransformer3DModel = _FakeTransformer
|
||||
diffusers.HunyuanVideo15Pipeline = _FakeHV15Pipeline
|
||||
diffusers.HunyuanVideo15Transformer3DModel = _FakeTransformer
|
||||
|
|
@ -1385,6 +1436,66 @@ def test_wan_a14b_cfg2_threaded_when_signature_has_it(fake_runtime):
|
|||
assert call2["guidance_scale_2"] is None
|
||||
|
||||
|
||||
def _tiny_png_data_url(width = 8, height = 8):
|
||||
import base64
|
||||
import io
|
||||
|
||||
from PIL import Image
|
||||
|
||||
buf = io.BytesIO()
|
||||
Image.new("RGB", (width, height), (128, 64, 32)).save(buf, format = "PNG")
|
||||
return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def test_wan_i2v_repo_is_trusted_and_detected():
|
||||
# The official -Diffusers repo loads as a full pipeline: it must be on the video
|
||||
# trust allowlist and resolve to the image-conditioned dual-expert family.
|
||||
backend = VideoBackend()
|
||||
fam = backend.validate_load_request(
|
||||
"Wan-AI/Wan2.2-I2V-A14B-Diffusers", model_kind = "pipeline"
|
||||
)
|
||||
assert fam.name == "wan2.2-i2v-a14b"
|
||||
assert fam.image_conditioned is True
|
||||
|
||||
|
||||
def test_wan_i2v_requires_image_and_threads_it(fake_runtime):
|
||||
# Loading the I2V family builds the dual-DiT image pipeline; generate() without a
|
||||
# source image is client input error, with one the decoded PIL image (resized to the
|
||||
# snapped output size) is threaded as the pipeline's ``image`` kwarg.
|
||||
backend = VideoBackend()
|
||||
status = backend.load_pipeline("Wan-AI/Wan2.2-I2V-A14B-Diffusers", model_kind = "pipeline")
|
||||
assert status["loaded"] is True and status["family"] == "wan2.2-i2v-a14b"
|
||||
assert status["image_input"] is True
|
||||
pipe = backend._state.pipe
|
||||
assert pipe.transformer is not None and pipe.transformer_2 is not None
|
||||
|
||||
with pytest.raises(ValueError, match = "image-to-video"):
|
||||
backend.generate(prompt = "a sloth")
|
||||
# begin_generate 400s synchronously too (no failed background job for a missing image),
|
||||
# and a raise must not leave the busy flag set.
|
||||
with pytest.raises(ValueError, match = "image-to-video"):
|
||||
backend.begin_generate(prompt = "a sloth")
|
||||
assert backend._generate_job_active is False
|
||||
|
||||
backend.generate(
|
||||
prompt = "a sloth", width = 832, height = 480, init_image = _tiny_png_data_url()
|
||||
)
|
||||
sent = pipe.last_kwargs["image"]
|
||||
assert sent is not None and sent.size == (832, 480)
|
||||
# The I2V defaults (40 steps / CFG 3.5, the card recipe) beat the generic wan 50/5.0.
|
||||
assert pipe.last_kwargs["num_inference_steps"] == 40
|
||||
assert pipe.last_kwargs["guidance_scale"] == 3.5
|
||||
|
||||
|
||||
def test_wan_t2v_rejects_source_image(fake_runtime):
|
||||
# A text-only family given an init_image must 400-fail loudly, not silently ignore it.
|
||||
backend = VideoBackend()
|
||||
status = backend.load_pipeline("Wan-AI/Wan2.2-T2V-A14B-Diffusers", model_kind = "pipeline")
|
||||
assert status["image_input"] is False
|
||||
with pytest.raises(ValueError, match = "does not take a source image"):
|
||||
backend.generate(prompt = "a sloth", init_image = _tiny_png_data_url())
|
||||
|
||||
|
||||
def test_wan_a14b_step_cache_applies_to_both_dits(fake_runtime):
|
||||
# A dual-DiT MoE load must engage the step cache on BOTH experts, not just the
|
||||
# first: transformer_2 handles the low-noise steps and would otherwise run uncached.
|
||||
|
|
|
|||
|
|
@ -86,12 +86,59 @@ def test_detect_wan_t2v_a14b(repo_id):
|
|||
assert fam.frame_step == 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"repo_id",
|
||||
[
|
||||
"Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
"wan-ai/wan2.2-i2v-a14b-diffusers",
|
||||
"Wan-AI/Wan2.2-I2V-A14B",
|
||||
"some/dir/wan2.2-i2v-a14b-Q4_K_M.gguf",
|
||||
],
|
||||
)
|
||||
def test_detect_wan_i2v_a14b(repo_id):
|
||||
# The I2V repo ids route to the image-conditioned dual-expert family: the pipeline is
|
||||
# WanImageToVideoPipeline and generate() requires a source image. Same DiT pair /
|
||||
# second guidance kwarg as T2V-A14B; boundary_ratio (0.9) lives in the pipeline config.
|
||||
fam = detect_video_family(repo_id)
|
||||
assert fam is not None and fam.name == "wan2.2-i2v-a14b"
|
||||
assert fam.pipeline_class == "WanImageToVideoPipeline"
|
||||
assert fam.transformer2_class == "WanTransformer3DModel"
|
||||
assert fam.is_moe is True
|
||||
assert fam.cfg2_kwarg == "guidance_scale_2"
|
||||
assert fam.image_conditioned is True
|
||||
assert fam.has_audio is False
|
||||
assert fam.frame_step == 4
|
||||
assert fam.vae_force_fp32 is True
|
||||
# Same shipped expert layout as T2V: the table holds the halved bf16-resident sizes.
|
||||
assert fam.bf16_components_gb == (57.2, 11.4, 0.5)
|
||||
|
||||
|
||||
def test_i2v_does_not_claim_t2v_ids_and_vice_versa():
|
||||
# "wan2.2-i2v" must not swallow the T2V/TI2V ids, and the generic wan families stay
|
||||
# text-only (image_conditioned False), so the image gate can't misfire on them.
|
||||
assert detect_video_family("Wan-AI/Wan2.2-T2V-A14B-Diffusers").name == "wan2.2-t2v-a14b"
|
||||
assert detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers").name == "wan2.2-ti2v-5b"
|
||||
assert detect_video_family("Wan-AI/Wan2.2-T2V-A14B-Diffusers").image_conditioned is False
|
||||
assert detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers").image_conditioned is False
|
||||
|
||||
|
||||
def test_wan_i2v_generation_defaults():
|
||||
# The I2V card recipe (40 steps, CFG 3.5) must beat the generic "wan" 50/5.0 key, and
|
||||
# the T2V ids must keep 50/5.0.
|
||||
assert default_video_generation_params(None, "Wan-AI/Wan2.2-I2V-A14B-Diffusers") == (40, 3.5)
|
||||
assert default_video_generation_params("wan2.2-i2v-a14b-Q4_K_M.gguf") == (40, 3.5)
|
||||
assert default_video_generation_params(None, "Wan-AI/Wan2.2-T2V-A14B-Diffusers") == (50, 5.0)
|
||||
assert default_video_generation_params(None, "Wan-AI/Wan2.2-TI2V-5B-Diffusers") == (50, 5.0)
|
||||
|
||||
|
||||
def test_detect_wan_overrides():
|
||||
# Short aliases the picker / GGUF filenames use resolve to the right family.
|
||||
assert detect_video_family("x", override = "wan2.2-5b").name == "wan2.2-ti2v-5b"
|
||||
assert detect_video_family("x", override = "wan-ti2v").name == "wan2.2-ti2v-5b"
|
||||
assert detect_video_family("x", override = "wan2.2-14b").name == "wan2.2-t2v-a14b"
|
||||
assert detect_video_family("x", override = "wan-t2v").name == "wan2.2-t2v-a14b"
|
||||
assert detect_video_family("x", override = "wan-i2v").name == "wan2.2-i2v-a14b"
|
||||
assert detect_video_family("x", override = "wan2.2-i2v").name == "wan2.2-i2v-a14b"
|
||||
|
||||
|
||||
def test_wan_and_ltx_do_not_cross_route():
|
||||
|
|
@ -150,6 +197,7 @@ def test_supported_names():
|
|||
"ltx-2",
|
||||
"wan2.2-ti2v-5b",
|
||||
"wan2.2-t2v-a14b",
|
||||
"wan2.2-i2v-a14b",
|
||||
"hunyuanvideo-1.5",
|
||||
"hunyuanvideo-1.5-720p",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -166,6 +166,8 @@ class _FakeBackend(video_module.VideoBackend):
|
|||
):
|
||||
if not self.loaded:
|
||||
raise RuntimeError(VIDEO_NOT_LOADED_MSG)
|
||||
# Recorded so route tests can assert request fields (init_image) reach the backend.
|
||||
self.last_generate_kwargs = dict(kwargs)
|
||||
return {
|
||||
"mp4_bytes": b"MP4-FAKE-BYTES",
|
||||
"seed": seed if seed is not None else 4242,
|
||||
|
|
@ -448,6 +450,19 @@ def test_generate_happy_path_persists_and_reports_record(client):
|
|||
assert fetched.content == b"MP4-FAKE-BYTES"
|
||||
|
||||
|
||||
def test_generate_threads_init_image_to_backend(client):
|
||||
# The image-to-video source image (init_image) must ride the request through
|
||||
# begin_generate into the backend generate call untouched.
|
||||
client.post(
|
||||
"/api/inference/video/load",
|
||||
json = {"model_path": "unsloth/LTX-2.3-GGUF", "gguf_filename": "q.gguf"},
|
||||
)
|
||||
data_url = "data:image/png;base64,aGk="
|
||||
_generate_and_wait(client, {"prompt": "a sloth", "seed": 3, "init_image": data_url})
|
||||
backend = video_module.get_video_backend()
|
||||
assert backend.last_generate_kwargs.get("init_image") == data_url
|
||||
|
||||
|
||||
def test_generate_without_load_returns_409(client):
|
||||
resp = client.post("/api/inference/video/generate", json = {"prompt": "p"})
|
||||
assert resp.status_code == 409
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue