diff --git a/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.check.ts b/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.check.ts index 7c073b978b..4050784d41 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.check.ts +++ b/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.check.ts @@ -196,6 +196,16 @@ for (const id of OLD_PIPELINE_MODELS) { assert.ok(got, `missing video load spec for ${id}`); assert.equal(got.kind, "pipeline", id); } +// Wan 2.2 I2V: pipeline load spec, and the canonical id groups with the -Diffusers artifact. +assert.equal( + loadSpecFor("Wan-AI/Wan2.2-I2V-A14B-Diffusers", VIDEO_CATALOG)?.kind, + "pipeline", +); +assert.equal( + groupForRepoId("Wan-AI/Wan2.2-I2V-A14B", VIDEO_CATALOG), + groupForRepoId("Wan-AI/Wan2.2-I2V-A14B-Diffusers", VIDEO_CATALOG), +); +assert.ok(groupForRepoId("Wan-AI/Wan2.2-I2V-A14B", VIDEO_CATALOG)); // GGUF artifacts report the gguf kind; unknown ids report null. assert.equal(loadSpecFor("unsloth/Z-Image-Turbo-GGUF", IMAGE_CATALOG)?.kind, "gguf"); assert.equal(loadSpecFor("someone/unknown", IMAGE_CATALOG), null); diff --git a/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.ts b/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.ts index 43a30fa5b2..aa1f5946b0 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.ts +++ b/studio/frontend/src/components/assistant-ui/model-selector/model-catalog.ts @@ -341,6 +341,15 @@ export const VIDEO_CATALOG: CatalogGroup[] = [ scope: "video", artifacts: [bf16Pipeline("Wan-AI/Wan2.2-T2V-A14B-Diffusers", 114)], }, + { + // Same dual-expert DiT pair as T2V-A14B, but the pipeline is image-to-video + // (WanImageToVideoPipeline): it animates a source image the video tab collects. + canonicalId: "Wan-AI/Wan2.2-I2V-A14B", + displayName: "Wan 2.2 I2V A14B (MoE)", + description: "Image-to-video, dual-expert", + scope: "video", + artifacts: [bf16Pipeline("Wan-AI/Wan2.2-I2V-A14B-Diffusers", 114)], + }, { canonicalId: "hunyuanvideo-community/HunyuanVideo-1.5", displayName: "HunyuanVideo 1.5", diff --git a/studio/frontend/src/features/video/api.ts b/studio/frontend/src/features/video/api.ts index a676068278..d34ac6c11f 100644 --- a/studio/frontend/src/features/video/api.ts +++ b/studio/frontend/src/features/video/api.ts @@ -49,6 +49,9 @@ export interface VideoStatus { transformer_quant?: string | null; // Whether the loaded family produces a synchronized audio track. has_audio: boolean; + // Whether the loaded family is image-to-video: the source-image control is shown and + // generate requires an image. + image_input?: boolean; // Per-family generation defaults + shape constraints; null when unloaded. defaults?: VideoGenerationDefaults | null; // Per-Advanced-control provenance, keyed by control name (memory_mode, speed_mode, @@ -144,6 +147,8 @@ export interface VideoGenerateRequest { steps?: number; guidance?: number; seed?: number; + // Source image (data URL) for image-to-video families; required by them, rejected elsewhere. + init_image?: string; } // A persisted clip's full generation recipe (the JSON sidecar of the MP4). diff --git a/studio/frontend/src/features/video/video-page.tsx b/studio/frontend/src/features/video/video-page.tsx index c634d5d1ea..caaac40735 100644 --- a/studio/frontend/src/features/video/video-page.tsx +++ b/studio/frontend/src/features/video/video-page.tsx @@ -89,7 +89,9 @@ const MODEL_DEFAULTS: Array<{ match: string; steps: number; guidance: number }> // "distilled" before the generic "ltx": the distilled model runs at 8 steps, guidance 1. { match: "distilled", steps: 8, guidance: 1 }, { match: "ltx", steps: 40, guidance: 4 }, - // Wan2.2 pipelines default to 50 steps at CFG 5.0 (WanPipeline defaults, verified in + // Wan2.2 I2V runs its card recipe (40 steps, CFG 3.5); before the generic "wan" key. + { match: "wan2.2-i2v", steps: 40, guidance: 3.5 }, + // Wan2.2 T2V pipelines default to 50 steps at CFG 5.0 (WanPipeline defaults, verified in // diffusers 0.39). The backend supplies the fps per family (24 for TI2V-5B, 16 for A14B). { match: "wan", steps: 50, guidance: 5 }, // HunyuanVideo-1.5 runs 50 steps; guidance 6 matches the guider the repo ships @@ -332,6 +334,88 @@ function Field({ ); } +// Source-image picker for image-to-video families: click or drag-drop an image, read it +// to a data URL the generate request sends as init_image. Mirrors the images tab's +// Transform dropzone (thumbnail preview + Clear once set). +function SourceImageDropzone({ + value, + onChange, +}: { + value: string | null; + onChange: (dataUrl: string | null) => void; +}) { + const inputRef = useRef(null); + const [dragging, setDragging] = useState(false); + + const readFile = useCallback( + (file: File | undefined | null) => { + if (!file || !file.type.startsWith("image/")) { + if (file) toast.error("Please choose an image file"); + return; + } + const reader = new FileReader(); + reader.onload = () => onChange(typeof reader.result === "string" ? reader.result : null); + reader.onerror = () => toast.error("Could not read the image"); + reader.readAsDataURL(file); + }, + [onChange], + ); + + if (value) { + return ( +
+ Source + +
+ ); + } + + return ( + + ); +} + // The engaged value of a resolved Advanced control, formatted for its "Auto: X" badge. // Short scheme/mode tokens go uppercase (FBCACHE); the attention backend the backend reports // as `_native_cudnn` shows as cuDNN. @@ -506,6 +590,9 @@ export function VideoPage({ active = true }: { active?: boolean }) { const [steps, setSteps] = useState(DEFAULT_GEN.steps); const [guidance, setGuidance] = useState(DEFAULT_GEN.guidance); const [seed, setSeed] = useState(""); + // Source image (data URL) for image-to-video families; the control renders only when the + // loaded family requires one (status.image_input). + const [initImage, setInitImage] = useState(null); // The chosen resolution preset index into the current preset list. const [resolutionIdx, setResolutionIdx] = useState(0); // The chosen frame count (must lie on the family's temporal lattice: k*frame_step+1). @@ -1211,6 +1298,11 @@ export function VideoPage({ active = true }: { active?: boolean }) { toast.error("Prompt is empty"); return; } + const needsImage = Boolean(status?.image_input); + if (needsImage && !initImage) { + toast.error("Attach a source image to animate"); + return; + } // Resolve a base seed up front. With an explicit seed the run is reproducible; with a // random one we still pick a concrete seed now so the recipe records it. let resolvedSeed: number | undefined; @@ -1247,6 +1339,8 @@ export function VideoPage({ active = true }: { active?: boolean }) { steps, guidance, seed: resolvedSeed, + // Only for image-to-video families; text-only families reject an image with a 400. + init_image: needsImage ? initImage ?? undefined : undefined, }); } catch (err) { if (!isMounted.current) return; @@ -1268,6 +1362,8 @@ export function VideoPage({ active = true }: { active?: boolean }) { numFrames, fps, steps, + status?.image_input, + initImage, startGenPoll, ]); @@ -1460,6 +1556,17 @@ export function VideoPage({ active = true }: { active?: boolean }) { /> + {/* Image-to-video families (Wan2.2-I2V) require a source image to animate; the + backend reports the capability via status.image_input. */} + {status?.image_input && ( + + + + )} + {/* A negative prompt only does anything with guidance on, so hide it at guidance 0 (the distilled model's default) instead of showing a dead field. */} {guidance > 0 && (