From cffee731354f88e417413cdc7434f743327c19c3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 17 Jul 2026 11:15:48 +0000 Subject: [PATCH] Add Wan2.2-I2V-A14B image-to-video support to the video backend Wan-AI/Wan2.2-I2V-A14B-Diffusers is the image-to-video sibling of the already supported T2V-A14B: the same dual-expert WanTransformer3DModel pair (boundary_ratio 0.9 in the pipeline config) behind WanImageToVideoPipeline, conditioning through the VAE latent (no CLIP-vision image encoder in this 2.2 variant). - New wan2.2-i2v-a14b family: image_conditioned flag, card recipe defaults (40 steps, CFG 3.5, 81 frames at 16 fps), the T2V memory table (57.2 GB both experts bf16), fp32-pinned VAE, and a wan2.2-i2v generation-defaults key ahead of the generic wan 50/5.0 entry. - Source-image plumbing: /video/generate takes init_image (base64/data URL); begin_generate 400s synchronously when an image-to-video family has no image or a text-only family is given one, and generate() decodes and resizes the image to the snapped output size before threading it as the pipeline's image kwarg. status() reports image_input so the UI can gate the control. - Trust the official -Diffusers repo for pipeline loads and transfer the measured wan quant recipes: fp8 keeps condition_embedder in bf16 (zero padding-row scale), mxfp8 and nvfp4 stay denied, the UMT5 auto TE quant resolves dense, and the balanced FBCache pin carries over. All tables share the T2V DiT profile. - Tests: family detection/aliases/defaults, the dual-DiT image pipeline load, the image gates on both begin_generate and generate, init_image route pass-through, and the quant exclude/deny/auto entries. GPU-verified on a B200: bf16 resident load (offload none, 72.8 GB peak) animating a conditioning image at 832x480/33f in 28.7 s with first-frame MAE 5.7 vs the source, and an int8 load with both experts quantized (43.1 GB peak, clean output). --- .../backend/core/inference/diffusion_cache.py | 2 + .../core/inference/diffusion_precision.py | 3 +- .../inference/diffusion_transformer_quant.py | 2 + studio/backend/core/inference/video.py | 46 ++++++++ .../backend/core/inference/video_families.py | 40 ++++++- studio/backend/models/inference.py | 12 ++ studio/backend/routes/video.py | 1 + .../tests/test_diffusion_transformer_quant.py | 3 + studio/backend/tests/test_video_backend.py | 111 ++++++++++++++++++ studio/backend/tests/test_video_families.py | 48 ++++++++ studio/backend/tests/test_video_routes.py | 15 +++ 11 files changed, 281 insertions(+), 2 deletions(-) diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.py index dfc3570329..fb34136773 100644 --- a/studio/backend/core/inference/diffusion_cache.py +++ b/studio/backend/core/inference/diffusion_cache.py @@ -89,6 +89,8 @@ _FBCACHE_QUALITY_THRESHOLDS: dict[str, tuple[float, float]] = { # 0.08 is the least-drift cache-on point, not a compliant one. _FAMILY_FBCACHE_THRESHOLDS: dict[tuple[str, str], tuple[float, float]] = { ("wan2.2-t2v-a14b", CQ_BALANCED): (DEFAULT_FBCACHE_THRESHOLD, DEFAULT_FBCACHE_THRESHOLD), + # I2V-A14B shares the T2V DiT pair; inherit the balanced pin (same drift profile). + ("wan2.2-i2v-a14b", CQ_BALANCED): (DEFAULT_FBCACHE_THRESHOLD, DEFAULT_FBCACHE_THRESHOLD), } diff --git a/studio/backend/core/inference/diffusion_precision.py b/studio/backend/core/inference/diffusion_precision.py index b1ac0e429b..afe9d04159 100644 --- a/studio/backend/core/inference/diffusion_precision.py +++ b/studio/backend/core/inference/diffusion_precision.py @@ -131,7 +131,8 @@ _TE_FAMILY_SCHEME_DENY: dict[str, frozenset[str]] = { # amplifies the perturbation ~3x harder than on wan2.2-ti2v-5b (0.0396, kept quantized there for # a real 1.09x on its faster DiT). _TE_AUTO_DENSE_FAMILIES: frozenset[str] = frozenset( - {"hunyuanvideo-1.5", "hunyuanvideo-1.5-720p", "wan2.2-t2v-a14b"} + # wan2.2-i2v-a14b inherits the T2V entry: same UMT5 encoder and MoE trajectory. + {"hunyuanvideo-1.5", "hunyuanvideo-1.5-720p", "wan2.2-t2v-a14b", "wan2.2-i2v-a14b"} ) # Map a TE torchao scheme to the transformer smoke-probe scheme (same GEMM), so ``auto`` degrades diff --git a/studio/backend/core/inference/diffusion_transformer_quant.py b/studio/backend/core/inference/diffusion_transformer_quant.py index 8c9a48785d..656edb61ef 100644 --- a/studio/backend/core/inference/diffusion_transformer_quant.py +++ b/studio/backend/core/inference/diffusion_transformer_quant.py @@ -127,6 +127,7 @@ _INT8_FAMILY_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = { _FP8_FAMILY_EXCLUDE_NAME_TOKENS: dict[str, tuple[str, ...]] = { "wan2.2-ti2v-5b": ("condition_embedder",), "wan2.2-t2v-a14b": ("condition_embedder",), # same DiT class + padded-text conditioning + "wan2.2-i2v-a14b": ("condition_embedder",), # same DiT class + padded-text conditioning } @@ -195,6 +196,7 @@ _FAMILY_SCHEME_DENY: dict[str, frozenset[str]] = { # (same scaled_mm family, not separately validated). 5B TI2V and A14B share the DiT + profile. "wan2.2-ti2v-5b": frozenset({TQ_MXFP8, TQ_NVFP4}), "wan2.2-t2v-a14b": frozenset({TQ_MXFP8, TQ_NVFP4}), + "wan2.2-i2v-a14b": frozenset({TQ_MXFP8, TQ_NVFP4}), # same DiT pair + quant profile as T2V # HunyuanVideo-1.5 DiT: fp8 renders black frames (LPIPS 0.82) and -- unlike Wan -- the failure # is NOT confinable to an input embedder: its MMDiT masks padding text tokens to zero inside # every block, so the context stream regenerates zero rows layer after layer (fp8 on only the diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py index 1a196504dd..95a84de10c 100644 --- a/studio/backend/core/inference/video.py +++ b/studio/backend/core/inference/video.py @@ -128,6 +128,7 @@ _TRUSTED_NON_GGUF_VIDEO_REPOS = frozenset( # Wan2.2 official diffusers base repos: safetensors-only, no remote code. "wan-ai/wan2.2-ti2v-5b-diffusers", "wan-ai/wan2.2-t2v-a14b-diffusers", + "wan-ai/wan2.2-i2v-a14b-diffusers", # HunyuanVideo-1.5 community Diffusers repacks (tencent's own repo is the # non-diffusers layout with no model_index.json, unloadable here). "hunyuanvideo-community/hunyuanvideo-1.5-diffusers-480p_t2v", @@ -1916,6 +1917,7 @@ class VideoBackend: guidance: Optional[float] = None, guidance_2: Optional[float] = None, seed: Optional[int] = None, + init_image: Optional[str] = None, ) -> None: """Validate cheaply, then run generate + gallery persist on a daemon thread. @@ -1934,6 +1936,21 @@ class VideoBackend: raise RuntimeError(VIDEO_NOT_LOADED_MSG) if self._generate_job_active: raise RuntimeError(VIDEO_GENERATION_BUSY_MSG) + # Image gate up front (cheap, no decode) so the POST 400s synchronously instead + # of reporting a failed job: an image-to-video family needs a source image, a + # text-only family has no ``image`` kwarg to take one. generate() re-checks for + # direct callers. getattr-guarded: tests fake _state without a family. + fam = getattr(self._state, "family", None) + if fam is not None: + if getattr(fam, "image_conditioned", False) and not (init_image or "").strip(): + raise ValueError( + f"{fam.name} is an image-to-video model: attach a source image " + "to animate." + ) + if init_image and not getattr(fam, "image_conditioned", False): + raise ValueError( + f"{fam.name} is a text-to-video model and does not take a source image." + ) # A background compile prewarm may hold _generate_lock. Signal its dedicated # cancel handle BEFORE registering ours so the real job preempts the warmup at # its next step boundary instead of queueing behind it (and so unload/cancel @@ -1963,6 +1980,7 @@ class VideoBackend: guidance = guidance, guidance_2 = guidance_2, seed = seed, + init_image = init_image, cancel_event = cancel, ), daemon = True, @@ -2074,6 +2092,7 @@ class VideoBackend: guidance: Optional[float] = None, guidance_2: Optional[float] = None, seed: Optional[int] = None, + init_image: Optional[str] = None, cancel_event: Optional[threading.Event] = None, ) -> dict[str, Any]: import torch @@ -2125,6 +2144,29 @@ class VideoBackend: "num_frames": frames, "generator": generator, } + # Image-conditioned families (WanImageToVideoPipeline) REQUIRE a source image; + # text-only families have no ``image`` kwarg to feed one to. Both mismatches are + # client input -> ValueError (the route/worker map it to a 400-style message). + if fam.image_conditioned: + if not (init_image or "").strip(): + raise ValueError( + f"{fam.name} is an image-to-video model: attach a source image " + "to animate." + ) + from .diffusion import _decode_b64_image + from PIL import Image + + init_pil = _decode_b64_image(init_image, mode = "RGB") + # The pipeline derives the latent grid from height/width and encodes the + # image at that size; resize here so the conditioning frame matches the + # snapped output size exactly (no center-crop surprises). + if init_pil.size != (width, height): + init_pil = init_pil.resize((width, height), Image.LANCZOS) + kwargs["image"] = init_pil + elif init_image: + raise ValueError( + f"{fam.name} is a text-to-video model and does not take a source image." + ) if fam.guidance_via_guider: # HunyuanVideo-1.5: __call__ has no guidance kwarg; CFG scale is a guider # attribute set per request (near-1 scales auto-disable CFG in the guider). @@ -2467,6 +2509,7 @@ class VideoBackend: "vae_quant": None, "cfg_parallel": None, "has_audio": False, + "image_input": False, "defaults": None, "resolved": None, } @@ -2497,6 +2540,9 @@ class VideoBackend: "vae_quant": state.vae_quant, "cfg_parallel": state.cfg_parallel, "has_audio": fam.has_audio, + # True for image-to-video families: the UI shows the source-image control and + # requires an image before submitting. + "image_input": fam.image_conditioned, "defaults": { "steps": default_steps, "guidance": default_guidance, diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py index 1c9b5ef3f7..a409cf77b2 100644 --- a/studio/backend/core/inference/video_families.py +++ b/studio/backend/core/inference/video_families.py @@ -79,6 +79,10 @@ class VideoFamily: vae_force_fp32: bool = False # Curated GGUF repo for the picker (the DiT as single-file GGUF quants). gguf_repo: Optional[str] = None + # True when the pipeline REQUIRES a conditioning image (WanImageToVideoPipeline): the + # generate path decodes/resizes the request's init_image and refuses a run without one; + # the UI shows the source-image control only for these families. + image_conditioned: bool = False _FAMILIES: tuple[VideoFamily, ...] = ( @@ -167,6 +171,38 @@ _FAMILIES: tuple[VideoFamily, ...] = ( vae_force_fp32 = True, # No gguf_repo: community GGUFs split the experts, and a single-file load covers only one. ), + # Wan2.2-I2V-A14B (diffusers >= 0.35, verified on 0.39): the image-to-video dual-expert MoE. + # Same DiT pair as T2V-A14B (boundary_ratio 0.9 vs T2V's 0.875, read from the pipeline + # config -- no plumbing), but the pipeline is WanImageToVideoPipeline and REQUIRES a + # conditioning image: this Wan2.2 variant conditions through the VAE latent (the repo's + # image_encoder/image_processor slots are null; no CLIP-vision like Wan2.1), so generate() + # threads the decoded init_image straight into ``image``. Card recipe: 40 steps, CFG 3.5, + # 81 frames at 16 fps. + VideoFamily( + name = "wan2.2-i2v-a14b", + pipeline_class = "WanImageToVideoPipeline", + transformer_class = "WanTransformer3DModel", + base_repo = "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + aliases = ("wan2.2-i2v", "wan-i2v", "wan-i2v-a14b"), + has_audio = False, + transformer2_class = "WanTransformer3DModel", + is_moe = True, + cfg2_kwarg = "guidance_scale_2", + image_conditioned = True, + # The I2V card runs 40 steps at CFG 3.5 (vs the T2V/pipeline default 50/5.0). + default_steps = 40, + default_guidance = 3.5, + default_num_frames = 81, + default_fps = 16, + frame_step = 4, + resolution_multiple = 16, + resolution_presets = ((1280, 720), (832, 480), (480, 832), (720, 1280)), + # Same shipped layout as T2V-A14B: each expert fp32 on disk (57.15 GB) -> ~28.6 bf16 + # each -> ~57.2 for both; UMT5 TE bf16 (11.4); VAE fp32 (0.5). + bf16_components_gb = (57.2, 11.4, 0.5), + vae_force_fp32 = True, + # No gguf_repo: community I2V GGUFs split the experts, and a single-file load covers only one. + ), # HunyuanVideo-1.5 (diffusers >= 0.39): 8.3B DiT, Qwen2.5-VL text encoder + ByT5 glyph # encoder. Three quirks: (1) __call__ has NO guidance kwarg; CFG on the ``guider`` # (guidance_via_guider); (2) NO callback_on_step_end (generate() uses the scheduler.step @@ -279,7 +315,9 @@ def snap_video_size(fam: VideoFamily, width: int, height: int) -> tuple[int, int _VIDEO_GENERATION_DEFAULTS: tuple[tuple[str, int, float], ...] = ( ("distilled", 8, 1.0), ("ltx", 40, 4.0), - # Wan2.2 pipelines default to 50 steps / CFG 5.0; both TI2V-5B and A14B share these. + # Wan2.2 I2V runs its card recipe (40 steps, CFG 3.5); must precede the generic "wan" key. + ("wan2.2-i2v", 40, 3.5), + # Wan2.2 T2V pipelines default to 50 steps / CFG 5.0; both TI2V-5B and T2V-A14B share these. ("wan", 50, 5.0), # HunyuanVideo-1.5: 50 steps with the guider's shipped CFG 6.0. ("hunyuanvideo", 50, 6.0), diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index b454dc4d79..1cb2a1a9a1 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -2552,6 +2552,13 @@ class VideoGenerateRequest(BaseModel): seed: Optional[int] = Field( None, ge = 0, le = 2**53 - 1, description = "Seed for reproducibility (random if omitted)" ) + # Image-to-video conditioning (base64 or data-URL). Required by image-conditioned + # families (Wan2.2-I2V); rejected with a 400 by text-only families. + init_image: Optional[str] = Field( + None, + description = "Source image to animate (base64/data-URL). Required for image-to-video " + "families; not accepted by text-to-video families.", + ) class GalleryVideo(BaseModel): @@ -2706,6 +2713,11 @@ class VideoStatusResponse(BaseModel): has_audio: bool = Field( False, description = "Whether the loaded family produces a synchronized audio track" ) + image_input: bool = Field( + False, + description = "Whether the loaded family requires a source image (image-to-video): the " + "UI shows the source-image control and generate refuses to run without one", + ) defaults: Optional[VideoGenerationDefaults] = Field( None, description = "Per-family generation defaults + shape constraints; null when unloaded" ) diff --git a/studio/backend/routes/video.py b/studio/backend/routes/video.py index 26dfdd8f38..60488de193 100644 --- a/studio/backend/routes/video.py +++ b/studio/backend/routes/video.py @@ -188,6 +188,7 @@ async def generate_video( guidance = request.guidance, guidance_2 = request.guidance_2, seed = request.seed, + init_image = request.init_image, ) except ValueError as exc: # Bad client input -- a 400 with the reason, not a generic 500. diff --git a/studio/backend/tests/test_diffusion_transformer_quant.py b/studio/backend/tests/test_diffusion_transformer_quant.py index e6d16cba57..e925cf14eb 100644 --- a/studio/backend/tests/test_diffusion_transformer_quant.py +++ b/studio/backend/tests/test_diffusion_transformer_quant.py @@ -453,6 +453,8 @@ def test_exclude_tokens_for_scheme_family(): assert exclude_tokens_for_scheme(TQ_FP8, "wan2.2-ti2v-5b") == ("condition_embedder",) assert exclude_tokens_for_scheme(TQ_FP8, "wan2.2-t2v-a14b") == ("condition_embedder",) + # I2V-A14B shares the T2V DiT pair, so the fp8 recipe transfers unchanged. + assert exclude_tokens_for_scheme(TQ_FP8, "wan2.2-i2v-a14b") == ("condition_embedder",) assert exclude_tokens_for_scheme(TQ_MXFP8, "wan2.2-ti2v-5b") == ("condition_embedder",) # Hunyuan is not localisable (fp8 stays denied), and an unknown family gets nothing. assert exclude_tokens_for_scheme(TQ_FP8, "hunyuanvideo-1.5") == () @@ -600,6 +602,7 @@ def test_family_allows_fp8_for_wan(monkeypatch): _allow(monkeypatch, {TQ_FP8, TQ_NVFP4, TQ_MXFP8, TQ_INT8}) assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-ti2v-5b") == TQ_FP8 assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-t2v-a14b") == TQ_FP8 + assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-i2v-a14b") == TQ_FP8 _allow(monkeypatch, {TQ_NVFP4, TQ_MXFP8, TQ_INT8}) # fp8 unavailable -> denied mx/nvfp4 skipped assert select_transformer_quant_scheme(_target(), "auto", family = "wan2.2-ti2v-5b") == TQ_INT8 diff --git a/studio/backend/tests/test_video_backend.py b/studio/backend/tests/test_video_backend.py index 454fd6b1ec..5782ad8e1b 100644 --- a/studio/backend/tests/test_video_backend.py +++ b/studio/backend/tests/test_video_backend.py @@ -276,6 +276,56 @@ class _FakeWanPipeMoE(_FakeWanPipeBase): return self._finish(num_inference_steps, num_frames, callback_on_step_end) +class _FakeWanI2VPipe(_FakeWanPipeBase): + """Dual-DiT image-to-video Wan pipeline (I2V-A14B): ``image`` IS in the signature + (WanImageToVideoPipeline.__call__ in diffusers 0.39) alongside guidance_scale_2.""" + + moe = True + + def __call__( + self, + *, + image = None, + prompt = None, + negative_prompt = None, + num_inference_steps = None, + guidance_scale = None, + guidance_scale_2 = None, + width = None, + height = None, + num_frames = None, + generator = None, + callback_on_step_end = None, + **kwargs, + ): + self.last_kwargs = { + "image": image, + "prompt": prompt, + "negative_prompt": negative_prompt, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "guidance_scale_2": guidance_scale_2, + "width": width, + "height": height, + "num_frames": num_frames, + **kwargs, + } + with self.transformer.cache_context("cond"): # real Wan pipeline wraps the denoise loop + pass + return self._finish(num_inference_steps, num_frames, callback_on_step_end) + + +class _FakeWanImageToVideoPipeline: + """WanImageToVideoPipeline fake (from_pretrained) for the I2V-A14B family.""" + + last: dict = {} + + @classmethod + def from_pretrained(cls, repo, **kwargs): + _FakeWanImageToVideoPipeline.last = {"repo": repo, **kwargs} + return _FakeWanI2VPipe() + + class _FakeWanPipelineSingle: """WanPipeline fake (from_pretrained). One class serves both families and picks the single-DiT / dual-DiT pipe by the repo id, exactly as diffusers dispatches on the @@ -387,6 +437,7 @@ def fake_runtime(monkeypatch): diffusers.LTX2VideoTransformer3DModel = _FakeTransformer # Wan2.2: one pipeline class serves both families (it dispatches on the repo id). diffusers.WanPipeline = _FakeWanPipelineSingle + diffusers.WanImageToVideoPipeline = _FakeWanImageToVideoPipeline diffusers.WanTransformer3DModel = _FakeTransformer diffusers.HunyuanVideo15Pipeline = _FakeHV15Pipeline diffusers.HunyuanVideo15Transformer3DModel = _FakeTransformer @@ -1385,6 +1436,66 @@ def test_wan_a14b_cfg2_threaded_when_signature_has_it(fake_runtime): assert call2["guidance_scale_2"] is None +def _tiny_png_data_url(width = 8, height = 8): + import base64 + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (width, height), (128, 64, 32)).save(buf, format = "PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + + +def test_wan_i2v_repo_is_trusted_and_detected(): + # The official -Diffusers repo loads as a full pipeline: it must be on the video + # trust allowlist and resolve to the image-conditioned dual-expert family. + backend = VideoBackend() + fam = backend.validate_load_request( + "Wan-AI/Wan2.2-I2V-A14B-Diffusers", model_kind = "pipeline" + ) + assert fam.name == "wan2.2-i2v-a14b" + assert fam.image_conditioned is True + + +def test_wan_i2v_requires_image_and_threads_it(fake_runtime): + # Loading the I2V family builds the dual-DiT image pipeline; generate() without a + # source image is client input error, with one the decoded PIL image (resized to the + # snapped output size) is threaded as the pipeline's ``image`` kwarg. + backend = VideoBackend() + status = backend.load_pipeline("Wan-AI/Wan2.2-I2V-A14B-Diffusers", model_kind = "pipeline") + assert status["loaded"] is True and status["family"] == "wan2.2-i2v-a14b" + assert status["image_input"] is True + pipe = backend._state.pipe + assert pipe.transformer is not None and pipe.transformer_2 is not None + + with pytest.raises(ValueError, match = "image-to-video"): + backend.generate(prompt = "a sloth") + # begin_generate 400s synchronously too (no failed background job for a missing image), + # and a raise must not leave the busy flag set. + with pytest.raises(ValueError, match = "image-to-video"): + backend.begin_generate(prompt = "a sloth") + assert backend._generate_job_active is False + + backend.generate( + prompt = "a sloth", width = 832, height = 480, init_image = _tiny_png_data_url() + ) + sent = pipe.last_kwargs["image"] + assert sent is not None and sent.size == (832, 480) + # The I2V defaults (40 steps / CFG 3.5, the card recipe) beat the generic wan 50/5.0. + assert pipe.last_kwargs["num_inference_steps"] == 40 + assert pipe.last_kwargs["guidance_scale"] == 3.5 + + +def test_wan_t2v_rejects_source_image(fake_runtime): + # A text-only family given an init_image must 400-fail loudly, not silently ignore it. + backend = VideoBackend() + status = backend.load_pipeline("Wan-AI/Wan2.2-T2V-A14B-Diffusers", model_kind = "pipeline") + assert status["image_input"] is False + with pytest.raises(ValueError, match = "does not take a source image"): + backend.generate(prompt = "a sloth", init_image = _tiny_png_data_url()) + + def test_wan_a14b_step_cache_applies_to_both_dits(fake_runtime): # A dual-DiT MoE load must engage the step cache on BOTH experts, not just the # first: transformer_2 handles the low-noise steps and would otherwise run uncached. diff --git a/studio/backend/tests/test_video_families.py b/studio/backend/tests/test_video_families.py index ab0a67ab4a..e742285c1b 100644 --- a/studio/backend/tests/test_video_families.py +++ b/studio/backend/tests/test_video_families.py @@ -86,12 +86,59 @@ def test_detect_wan_t2v_a14b(repo_id): assert fam.frame_step == 4 +@pytest.mark.parametrize( + "repo_id", + [ + "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "wan-ai/wan2.2-i2v-a14b-diffusers", + "Wan-AI/Wan2.2-I2V-A14B", + "some/dir/wan2.2-i2v-a14b-Q4_K_M.gguf", + ], +) +def test_detect_wan_i2v_a14b(repo_id): + # The I2V repo ids route to the image-conditioned dual-expert family: the pipeline is + # WanImageToVideoPipeline and generate() requires a source image. Same DiT pair / + # second guidance kwarg as T2V-A14B; boundary_ratio (0.9) lives in the pipeline config. + fam = detect_video_family(repo_id) + assert fam is not None and fam.name == "wan2.2-i2v-a14b" + assert fam.pipeline_class == "WanImageToVideoPipeline" + assert fam.transformer2_class == "WanTransformer3DModel" + assert fam.is_moe is True + assert fam.cfg2_kwarg == "guidance_scale_2" + assert fam.image_conditioned is True + assert fam.has_audio is False + assert fam.frame_step == 4 + assert fam.vae_force_fp32 is True + # Same shipped expert layout as T2V: the table holds the halved bf16-resident sizes. + assert fam.bf16_components_gb == (57.2, 11.4, 0.5) + + +def test_i2v_does_not_claim_t2v_ids_and_vice_versa(): + # "wan2.2-i2v" must not swallow the T2V/TI2V ids, and the generic wan families stay + # text-only (image_conditioned False), so the image gate can't misfire on them. + assert detect_video_family("Wan-AI/Wan2.2-T2V-A14B-Diffusers").name == "wan2.2-t2v-a14b" + assert detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers").name == "wan2.2-ti2v-5b" + assert detect_video_family("Wan-AI/Wan2.2-T2V-A14B-Diffusers").image_conditioned is False + assert detect_video_family("Wan-AI/Wan2.2-TI2V-5B-Diffusers").image_conditioned is False + + +def test_wan_i2v_generation_defaults(): + # The I2V card recipe (40 steps, CFG 3.5) must beat the generic "wan" 50/5.0 key, and + # the T2V ids must keep 50/5.0. + assert default_video_generation_params(None, "Wan-AI/Wan2.2-I2V-A14B-Diffusers") == (40, 3.5) + assert default_video_generation_params("wan2.2-i2v-a14b-Q4_K_M.gguf") == (40, 3.5) + assert default_video_generation_params(None, "Wan-AI/Wan2.2-T2V-A14B-Diffusers") == (50, 5.0) + assert default_video_generation_params(None, "Wan-AI/Wan2.2-TI2V-5B-Diffusers") == (50, 5.0) + + def test_detect_wan_overrides(): # Short aliases the picker / GGUF filenames use resolve to the right family. assert detect_video_family("x", override = "wan2.2-5b").name == "wan2.2-ti2v-5b" assert detect_video_family("x", override = "wan-ti2v").name == "wan2.2-ti2v-5b" assert detect_video_family("x", override = "wan2.2-14b").name == "wan2.2-t2v-a14b" assert detect_video_family("x", override = "wan-t2v").name == "wan2.2-t2v-a14b" + assert detect_video_family("x", override = "wan-i2v").name == "wan2.2-i2v-a14b" + assert detect_video_family("x", override = "wan2.2-i2v").name == "wan2.2-i2v-a14b" def test_wan_and_ltx_do_not_cross_route(): @@ -150,6 +197,7 @@ def test_supported_names(): "ltx-2", "wan2.2-ti2v-5b", "wan2.2-t2v-a14b", + "wan2.2-i2v-a14b", "hunyuanvideo-1.5", "hunyuanvideo-1.5-720p", ) diff --git a/studio/backend/tests/test_video_routes.py b/studio/backend/tests/test_video_routes.py index e4602a658d..3d8e00c6b2 100644 --- a/studio/backend/tests/test_video_routes.py +++ b/studio/backend/tests/test_video_routes.py @@ -166,6 +166,8 @@ class _FakeBackend(video_module.VideoBackend): ): if not self.loaded: raise RuntimeError(VIDEO_NOT_LOADED_MSG) + # Recorded so route tests can assert request fields (init_image) reach the backend. + self.last_generate_kwargs = dict(kwargs) return { "mp4_bytes": b"MP4-FAKE-BYTES", "seed": seed if seed is not None else 4242, @@ -448,6 +450,19 @@ def test_generate_happy_path_persists_and_reports_record(client): assert fetched.content == b"MP4-FAKE-BYTES" +def test_generate_threads_init_image_to_backend(client): + # The image-to-video source image (init_image) must ride the request through + # begin_generate into the backend generate call untouched. + client.post( + "/api/inference/video/load", + json = {"model_path": "unsloth/LTX-2.3-GGUF", "gguf_filename": "q.gguf"}, + ) + data_url = "data:image/png;base64,aGk=" + _generate_and_wait(client, {"prompt": "a sloth", "seed": 3, "init_image": data_url}) + backend = video_module.get_video_backend() + assert backend.last_generate_kwargs.get("init_image") == data_url + + def test_generate_without_load_returns_409(client): resp = client.post("/api/inference/video/generate", json = {"prompt": "p"}) assert resp.status_code == 409