Surface Wan2.2-I2V-A14B in the video tab with a source-image control
- Catalog: a Wan 2.2 I2V A14B group (bf16 pipeline artifact, 114 GB estimate) next to the T2V entry, with check assertions for the load spec and canonical grouping. - Video tab: a source-image dropzone (click or drag-drop, thumbnail preview + clear) shown only when the loaded family reports image_input; Generate refuses to submit without an image and sends it as init_image. The wan2.2-i2v defaults key seeds the card recipe (40 steps, CFG 3.5) ahead of the generic wan entry. - api.ts: image_input on VideoStatus, init_image on VideoGenerateRequest.
This commit is contained in:
parent
cffee73135
commit
b13d5e3573
4 changed files with 132 additions and 1 deletions
|
|
@ -196,6 +196,16 @@ for (const id of OLD_PIPELINE_MODELS) {
|
|||
assert.ok(got, `missing video load spec for ${id}`);
|
||||
assert.equal(got.kind, "pipeline", id);
|
||||
}
|
||||
// Wan 2.2 I2V: pipeline load spec, and the canonical id groups with the -Diffusers artifact.
|
||||
assert.equal(
|
||||
loadSpecFor("Wan-AI/Wan2.2-I2V-A14B-Diffusers", VIDEO_CATALOG)?.kind,
|
||||
"pipeline",
|
||||
);
|
||||
assert.equal(
|
||||
groupForRepoId("Wan-AI/Wan2.2-I2V-A14B", VIDEO_CATALOG),
|
||||
groupForRepoId("Wan-AI/Wan2.2-I2V-A14B-Diffusers", VIDEO_CATALOG),
|
||||
);
|
||||
assert.ok(groupForRepoId("Wan-AI/Wan2.2-I2V-A14B", VIDEO_CATALOG));
|
||||
// GGUF artifacts report the gguf kind; unknown ids report null.
|
||||
assert.equal(loadSpecFor("unsloth/Z-Image-Turbo-GGUF", IMAGE_CATALOG)?.kind, "gguf");
|
||||
assert.equal(loadSpecFor("someone/unknown", IMAGE_CATALOG), null);
|
||||
|
|
|
|||
|
|
@ -341,6 +341,15 @@ export const VIDEO_CATALOG: CatalogGroup[] = [
|
|||
scope: "video",
|
||||
artifacts: [bf16Pipeline("Wan-AI/Wan2.2-T2V-A14B-Diffusers", 114)],
|
||||
},
|
||||
{
|
||||
// Same dual-expert DiT pair as T2V-A14B, but the pipeline is image-to-video
|
||||
// (WanImageToVideoPipeline): it animates a source image the video tab collects.
|
||||
canonicalId: "Wan-AI/Wan2.2-I2V-A14B",
|
||||
displayName: "Wan 2.2 I2V A14B (MoE)",
|
||||
description: "Image-to-video, dual-expert",
|
||||
scope: "video",
|
||||
artifacts: [bf16Pipeline("Wan-AI/Wan2.2-I2V-A14B-Diffusers", 114)],
|
||||
},
|
||||
{
|
||||
canonicalId: "hunyuanvideo-community/HunyuanVideo-1.5",
|
||||
displayName: "HunyuanVideo 1.5",
|
||||
|
|
|
|||
|
|
@ -49,6 +49,9 @@ export interface VideoStatus {
|
|||
transformer_quant?: string | null;
|
||||
// Whether the loaded family produces a synchronized audio track.
|
||||
has_audio: boolean;
|
||||
// Whether the loaded family is image-to-video: the source-image control is shown and
|
||||
// generate requires an image.
|
||||
image_input?: boolean;
|
||||
// Per-family generation defaults + shape constraints; null when unloaded.
|
||||
defaults?: VideoGenerationDefaults | null;
|
||||
// Per-Advanced-control provenance, keyed by control name (memory_mode, speed_mode,
|
||||
|
|
@ -144,6 +147,8 @@ export interface VideoGenerateRequest {
|
|||
steps?: number;
|
||||
guidance?: number;
|
||||
seed?: number;
|
||||
// Source image (data URL) for image-to-video families; required by them, rejected elsewhere.
|
||||
init_image?: string;
|
||||
}
|
||||
|
||||
// A persisted clip's full generation recipe (the JSON sidecar of the MP4).
|
||||
|
|
|
|||
|
|
@ -89,7 +89,9 @@ const MODEL_DEFAULTS: Array<{ match: string; steps: number; guidance: number }>
|
|||
// "distilled" before the generic "ltx": the distilled model runs at 8 steps, guidance 1.
|
||||
{ match: "distilled", steps: 8, guidance: 1 },
|
||||
{ match: "ltx", steps: 40, guidance: 4 },
|
||||
// Wan2.2 pipelines default to 50 steps at CFG 5.0 (WanPipeline defaults, verified in
|
||||
// Wan2.2 I2V runs its card recipe (40 steps, CFG 3.5); before the generic "wan" key.
|
||||
{ match: "wan2.2-i2v", steps: 40, guidance: 3.5 },
|
||||
// Wan2.2 T2V pipelines default to 50 steps at CFG 5.0 (WanPipeline defaults, verified in
|
||||
// diffusers 0.39). The backend supplies the fps per family (24 for TI2V-5B, 16 for A14B).
|
||||
{ match: "wan", steps: 50, guidance: 5 },
|
||||
// HunyuanVideo-1.5 runs 50 steps; guidance 6 matches the guider the repo ships
|
||||
|
|
@ -332,6 +334,88 @@ function Field({
|
|||
);
|
||||
}
|
||||
|
||||
// Source-image picker for image-to-video families: click or drag-drop an image, read it
|
||||
// to a data URL the generate request sends as init_image. Mirrors the images tab's
|
||||
// Transform dropzone (thumbnail preview + Clear once set).
|
||||
function SourceImageDropzone({
|
||||
value,
|
||||
onChange,
|
||||
}: {
|
||||
value: string | null;
|
||||
onChange: (dataUrl: string | null) => void;
|
||||
}) {
|
||||
const inputRef = useRef<HTMLInputElement | null>(null);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
|
||||
const readFile = useCallback(
|
||||
(file: File | undefined | null) => {
|
||||
if (!file || !file.type.startsWith("image/")) {
|
||||
if (file) toast.error("Please choose an image file");
|
||||
return;
|
||||
}
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => onChange(typeof reader.result === "string" ? reader.result : null);
|
||||
reader.onerror = () => toast.error("Could not read the image");
|
||||
reader.readAsDataURL(file);
|
||||
},
|
||||
[onChange],
|
||||
);
|
||||
|
||||
if (value) {
|
||||
return (
|
||||
<div className="relative overflow-hidden rounded-xl border border-border">
|
||||
<img src={value} alt="Source" className="max-h-44 w-full object-contain bg-muted/30" />
|
||||
<Button
|
||||
type="button"
|
||||
variant="secondary"
|
||||
size="icon"
|
||||
aria-label="Remove source image"
|
||||
title="Remove"
|
||||
className="absolute right-1.5 top-1.5 size-7"
|
||||
onClick={() => {
|
||||
onChange(null);
|
||||
if (inputRef.current) inputRef.current.value = "";
|
||||
}}
|
||||
>
|
||||
<HugeiconsIcon icon={Delete02Icon} className="size-3.5" />
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => inputRef.current?.click()}
|
||||
onDragOver={(e) => {
|
||||
e.preventDefault();
|
||||
setDragging(true);
|
||||
}}
|
||||
onDragLeave={() => setDragging(false)}
|
||||
onDrop={(e) => {
|
||||
e.preventDefault();
|
||||
setDragging(false);
|
||||
readFile(e.dataTransfer.files?.[0]);
|
||||
}}
|
||||
className={cn(
|
||||
"flex h-24 w-full items-center justify-center rounded-xl border border-dashed text-xs",
|
||||
dragging
|
||||
? "border-primary/60 bg-primary/5 text-foreground"
|
||||
: "border-border text-muted-foreground hover:bg-muted/40",
|
||||
)}
|
||||
>
|
||||
Click or drop an image to animate
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="file"
|
||||
accept="image/*"
|
||||
className="hidden"
|
||||
onChange={(e) => readFile(e.target.files?.[0])}
|
||||
/>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
// The engaged value of a resolved Advanced control, formatted for its "Auto: X" badge.
|
||||
// Short scheme/mode tokens go uppercase (FBCACHE); the attention backend the backend reports
|
||||
// as `_native_cudnn` shows as cuDNN.
|
||||
|
|
@ -506,6 +590,9 @@ export function VideoPage({ active = true }: { active?: boolean }) {
|
|||
const [steps, setSteps] = useState(DEFAULT_GEN.steps);
|
||||
const [guidance, setGuidance] = useState(DEFAULT_GEN.guidance);
|
||||
const [seed, setSeed] = useState("");
|
||||
// Source image (data URL) for image-to-video families; the control renders only when the
|
||||
// loaded family requires one (status.image_input).
|
||||
const [initImage, setInitImage] = useState<string | null>(null);
|
||||
// The chosen resolution preset index into the current preset list.
|
||||
const [resolutionIdx, setResolutionIdx] = useState(0);
|
||||
// The chosen frame count (must lie on the family's temporal lattice: k*frame_step+1).
|
||||
|
|
@ -1211,6 +1298,11 @@ export function VideoPage({ active = true }: { active?: boolean }) {
|
|||
toast.error("Prompt is empty");
|
||||
return;
|
||||
}
|
||||
const needsImage = Boolean(status?.image_input);
|
||||
if (needsImage && !initImage) {
|
||||
toast.error("Attach a source image to animate");
|
||||
return;
|
||||
}
|
||||
// Resolve a base seed up front. With an explicit seed the run is reproducible; with a
|
||||
// random one we still pick a concrete seed now so the recipe records it.
|
||||
let resolvedSeed: number | undefined;
|
||||
|
|
@ -1247,6 +1339,8 @@ export function VideoPage({ active = true }: { active?: boolean }) {
|
|||
steps,
|
||||
guidance,
|
||||
seed: resolvedSeed,
|
||||
// Only for image-to-video families; text-only families reject an image with a 400.
|
||||
init_image: needsImage ? initImage ?? undefined : undefined,
|
||||
});
|
||||
} catch (err) {
|
||||
if (!isMounted.current) return;
|
||||
|
|
@ -1268,6 +1362,8 @@ export function VideoPage({ active = true }: { active?: boolean }) {
|
|||
numFrames,
|
||||
fps,
|
||||
steps,
|
||||
status?.image_input,
|
||||
initImage,
|
||||
startGenPoll,
|
||||
]);
|
||||
|
||||
|
|
@ -1460,6 +1556,17 @@ export function VideoPage({ active = true }: { active?: boolean }) {
|
|||
/>
|
||||
</Field>
|
||||
|
||||
{/* Image-to-video families (Wan2.2-I2V) require a source image to animate; the
|
||||
backend reports the capability via status.image_input. */}
|
||||
{status?.image_input && (
|
||||
<Field
|
||||
label="Source image"
|
||||
hint="The image to animate. The clip starts from this frame; it is resized to the selected resolution."
|
||||
>
|
||||
<SourceImageDropzone value={initImage} onChange={setInitImage} />
|
||||
</Field>
|
||||
)}
|
||||
|
||||
{/* A negative prompt only does anything with guidance on, so hide it at guidance 0
|
||||
(the distilled model's default) instead of showing a dead field. */}
|
||||
{guidance > 0 && (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue