Add Video tab to Studio frontend
Add a Video generation page that mirrors the Images feature's create workflow. It loads a text-to-video model, generates a clip, and plays it back inline with the gallery of past clips. - src/features/video/api.ts: typed client for the /api/inference/video routes (load, load-progress, generate, generate-progress, cancel, status, unload, gallery CRUD, and an auth-protected MP4 blob fetch). - src/features/video/video-page.tsx: the page. Curated model picker (LTX 2.3 distilled GGUF, LTX 2 base pipeline), prompt and negative prompt, resolution preset select, duration select over the family's temporal lattice, fixed fps display, steps and guidance sliders seeded from per-model defaults, seed box. Generate polls per-step progress with a phase label and ETA and a Cancel button, then plays the result in a video player with a download button and an audio badge. Gallery strip below with per-card delete and clear all. Right-docked Advanced panel for memory, speed, attention, and step-cache with Auto badges fed from the resolved status. - Register the page: router child, /video route, sidebar nav item with a video icon after Images, and the __root keep-alive mount so an in-flight generation survives leaving the tab. - Add the text-to-video task to the model picker so video models never appear in the chat picker. - Add the video nav label to the en and zh-CN locales.
This commit is contained in:
parent
e24df85f12
commit
a487f3c1c0
10 changed files with 1789 additions and 9 deletions
|
|
@ -11,6 +11,7 @@ import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId";
|
|||
import { Route as chatRoute } from "./routes/chat";
|
||||
import { Route as exportRoute } from "./routes/export";
|
||||
import { Route as imagesRoute } from "./routes/images";
|
||||
import { Route as videoRoute } from "./routes/video";
|
||||
import { Route as indexRoute } from "./routes/index";
|
||||
import { Route as loginRoute } from "./routes/login";
|
||||
import { Route as hubRoute } from "./routes/hub";
|
||||
|
|
@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([
|
|||
projectsRoute,
|
||||
exportRoute,
|
||||
imagesRoute,
|
||||
videoRoute,
|
||||
dataRecipesRoute,
|
||||
dataRecipeRoute,
|
||||
]);
|
||||
|
|
|
|||
|
|
@ -65,6 +65,13 @@ const ImagesPage = lazy(() =>
|
|||
import("@/features/images").then((m) => ({ default: m.ImagesPage })),
|
||||
);
|
||||
|
||||
// VideoPage gets the same persistent-mount treatment as ImagesPage so an in-flight
|
||||
// generation survives leaving the tab. Kept lazy so its bundle loads only on the first
|
||||
// /video visit.
|
||||
const VideoPage = lazy(() =>
|
||||
import("@/features/video").then((m) => ({ default: m.VideoPage })),
|
||||
);
|
||||
|
||||
function PersonalizationSyncMount() {
|
||||
usePersonalizationSync(hasAuthToken());
|
||||
return null;
|
||||
|
|
@ -167,13 +174,24 @@ function RootLayout() {
|
|||
setImagesMounted(true);
|
||||
}
|
||||
const shouldMountImages = isImagesRoute || imagesMounted;
|
||||
// Chat and Images both render their own full-height shell (a fixed top rail + an
|
||||
// internally-scrolling body), so both want the chat-style layout: no outer pt-14
|
||||
// inset and no outer scroll. Keying the layout off isChatRoute alone gave /images
|
||||
// the non-chat pt-14 + outer overflow, pushing its picker down and clipping the
|
||||
|
||||
// Same persistent-mount treatment for /video so a long generation keeps running when
|
||||
// the user flips to another tab (VideoPage reads no URL search, so it needs no freeze
|
||||
// dance -- just the mount latch). Mounts lazily on first /video visit, then stays
|
||||
// mounted, hidden+inert while off-route.
|
||||
const isVideoRoute = pathname === "/video";
|
||||
const [videoMounted, setVideoMounted] = useState(isVideoRoute);
|
||||
if (isVideoRoute && !videoMounted) {
|
||||
setVideoMounted(true);
|
||||
}
|
||||
const shouldMountVideo = isVideoRoute || videoMounted;
|
||||
// Chat, Images and Video all render their own full-height shell (a fixed top rail + an
|
||||
// internally-scrolling body), so all three want the chat-style layout: no outer pt-14
|
||||
// inset and no outer scroll. Keying the layout off isChatRoute alone gave /images and
|
||||
// /video the non-chat pt-14 + outer overflow, pushing the picker down and clipping the
|
||||
// bottom gallery. Treat them the same for the container padding/overflow only; the
|
||||
// keep-alive mounts below stay keyed to each specific route.
|
||||
const isChatLike = isChatRoute || isImagesRoute;
|
||||
const isChatLike = isChatRoute || isImagesRoute || isVideoRoute;
|
||||
|
||||
useTrainingUnloadGuard();
|
||||
// Global export driver: streams worker logs and tracks status from any route
|
||||
|
|
@ -305,12 +323,29 @@ function RootLayout() {
|
|||
</Suspense>
|
||||
</div>
|
||||
)}
|
||||
{/* Same keep-alive treatment for Video so a long generation keeps running
|
||||
off-tab; `active` force-closes its body-portaled overlays (model selector,
|
||||
recipe popover) so none can bleed over another tab while hidden. */}
|
||||
{shouldMountVideo && (
|
||||
<div
|
||||
className={
|
||||
isVideoRoute
|
||||
? "flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-hidden"
|
||||
: "hidden"
|
||||
}
|
||||
inert={!isVideoRoute || undefined}
|
||||
>
|
||||
<Suspense fallback={<RouteFallback />}>
|
||||
<VideoPage active={isVideoRoute} />
|
||||
</Suspense>
|
||||
</div>
|
||||
)}
|
||||
{/* Use mode="popLayout" instead of "wait" to prevent UI freezes when
|
||||
switching from heavy pages (like Export with many checkpoints).
|
||||
"popLayout" allows the new route to mount immediately while the
|
||||
old one animates out, avoiding blocking on expensive exit renders.
|
||||
See issue #5850. */}
|
||||
{!isChatRoute && !isImagesRoute && (
|
||||
{!isChatRoute && !isImagesRoute && !isVideoRoute && (
|
||||
<AnimatePresence initial={false} mode="popLayout">
|
||||
<motion.div
|
||||
key={pathname}
|
||||
|
|
|
|||
16
studio/frontend/src/app/routes/video.tsx
Normal file
16
studio/frontend/src/app/routes/video.tsx
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { createRoute } from "@tanstack/react-router";
|
||||
import { requireAuth } from "../auth-guards";
|
||||
import { Route as rootRoute } from "./__root";
|
||||
|
||||
// RootLayout renders VideoPage persistently (so an in-flight generation is not cancelled
|
||||
// when leaving the tab); this route only owns the URL + auth gate.
|
||||
export const Route = createRoute({
|
||||
getParentRoute: () => rootRoute,
|
||||
path: "/video",
|
||||
staticData: { title: "Video" },
|
||||
beforeLoad: () => requireAuth(),
|
||||
component: () => null,
|
||||
});
|
||||
|
|
@ -80,6 +80,7 @@ import {
|
|||
Settings02Icon,
|
||||
Sun03Icon,
|
||||
TestTube01Icon,
|
||||
Video01Icon,
|
||||
ZapIcon,
|
||||
} from "@hugeicons/core-free-icons";
|
||||
import {
|
||||
|
|
@ -1207,6 +1208,15 @@ export function AppSidebar() {
|
|||
closeMobileIfOpen();
|
||||
}}
|
||||
/>
|
||||
<NavItem
|
||||
icon={Video01Icon}
|
||||
label={t("shell.navigation.video")}
|
||||
active={pathname === "/video" || pathname.startsWith("/video/")}
|
||||
onClick={() => {
|
||||
navigate({ to: "/video" });
|
||||
closeMobileIfOpen();
|
||||
}}
|
||||
/>
|
||||
{/* Train has a labelled section when expanded; plain icon here only when collapsed. */}
|
||||
<NavItem
|
||||
icon={TestTubeOutlineIcon}
|
||||
|
|
|
|||
|
|
@ -1053,15 +1053,25 @@ export const IMAGE_GEN_TASKS = [
|
|||
"image-text-to-image",
|
||||
] as const;
|
||||
|
||||
// Video-generation pipeline tasks: handled by the Video page, never loadable as
|
||||
// chat models. The backend reports "text-to-video" for video-diffusion GGUFs. The
|
||||
// Video page reuses this as its picker's `task` filter, so it lives here.
|
||||
export const VIDEO_GEN_TASKS = ["text-to-video"] as const;
|
||||
|
||||
// Diffusion GGUF archs the Images backend can't assemble yet (SD/SDXL/PixArt/Wan/
|
||||
// ...). The backend tags them with this task so the chat picker hides them -- they
|
||||
// die with "unknown model architecture" in llama.cpp -- while the Images picker,
|
||||
// which filters on IMAGE_GEN_TASKS, also leaves them out (they'd 400 on load).
|
||||
const UNSUPPORTED_DIFFUSION_TASK = "image-diffusion-unsupported";
|
||||
|
||||
// Tasks that must never appear as a loadable chat model: the Images-handled
|
||||
// generation tasks plus the non-loadable diffusion tag above.
|
||||
const NON_CHAT_TASKS: readonly string[] = [...IMAGE_GEN_TASKS, UNSUPPORTED_DIFFUSION_TASK];
|
||||
// Tasks that must never appear as a loadable chat model: the Images- and Video-handled
|
||||
// generation tasks plus the non-loadable diffusion tag above. Keeping text-to-video here
|
||||
// stops a downloaded video GGUF from showing up as a loadable chat model (it would 400).
|
||||
const NON_CHAT_TASKS: readonly string[] = [
|
||||
...IMAGE_GEN_TASKS,
|
||||
...VIDEO_GEN_TASKS,
|
||||
UNSUPPORTED_DIFFUSION_TASK,
|
||||
];
|
||||
|
||||
// Editing/inpaint checkpoints are tagged image-to-image but need an input image,
|
||||
// which the text-to-image backend rejects (mirrors its _EDIT_KEYWORDS). Hidden by
|
||||
|
|
|
|||
226
studio/frontend/src/features/video/api.ts
Normal file
226
studio/frontend/src/features/video/api.ts
Normal file
|
|
@ -0,0 +1,226 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
import { authFetch } from "@/features/auth";
|
||||
import { readFastApiError } from "@/lib/format-fastapi-error";
|
||||
|
||||
// One Advanced control's resolved value + provenance, for the "Auto: X" badges. Same
|
||||
// shape the diffusion status uses. `value` is the engaged value (a scheme/mode string,
|
||||
// null when off, or a boolean); `source` is "auto" (this backend decided) or "explicit"
|
||||
// (the caller set it); `reason` is the short why shown as a tooltip.
|
||||
export interface VideoResolvedControl {
|
||||
value: string | boolean | null;
|
||||
source: "auto" | "explicit";
|
||||
reason: string;
|
||||
}
|
||||
|
||||
// Per-family generation defaults + shape constraints, from status.defaults when loaded.
|
||||
export interface VideoGenerationDefaults {
|
||||
steps: number;
|
||||
guidance: number;
|
||||
num_frames: number;
|
||||
fps: number;
|
||||
// Temporal lattice: valid frame counts are k * frame_step + 1.
|
||||
frame_step: number;
|
||||
// Width/height must be divisible by this.
|
||||
resolution_multiple: number;
|
||||
// (width, height) presets the UI offers, default first.
|
||||
resolution_presets: Array<[number, number]>;
|
||||
}
|
||||
|
||||
export interface VideoStatus {
|
||||
loaded: boolean;
|
||||
repo_id: string | null;
|
||||
family: string | null;
|
||||
base_repo: string | null;
|
||||
device: string | null;
|
||||
dtype: string | null;
|
||||
// Resolved load kind: "gguf" | "single_file" | "pipeline". Null when not loaded.
|
||||
model_kind?: string | null;
|
||||
// Resolved offload policy: none | group | model | sequential.
|
||||
offload_policy?: string | null;
|
||||
vae_tiling: boolean;
|
||||
memory_mode?: string | null;
|
||||
speed_mode?: string | null;
|
||||
// Speed optimisations actually engaged.
|
||||
speed_optims: string[];
|
||||
attention_backend?: string | null;
|
||||
transformer_cache?: string | null;
|
||||
// Whether the loaded family produces a synchronized audio track.
|
||||
has_audio: boolean;
|
||||
// Per-family generation defaults + shape constraints; null when unloaded.
|
||||
defaults?: VideoGenerationDefaults | null;
|
||||
// Per-Advanced-control provenance, keyed by control name (memory_mode, speed_mode,
|
||||
// attention_backend, transformer_cache). The "Auto: X" badges read it. Null when
|
||||
// nothing is loaded or on a backend that doesn't record it.
|
||||
resolved?: Record<string, VideoResolvedControl> | null;
|
||||
}
|
||||
|
||||
export interface VideoGenerateProgress {
|
||||
active: boolean;
|
||||
// "denoise" | "export" | null.
|
||||
phase?: string | null;
|
||||
step: number;
|
||||
total: number;
|
||||
eta_seconds?: number | null;
|
||||
}
|
||||
|
||||
export interface VideoLoadProgress {
|
||||
phase: "downloading" | "finalizing" | "ready" | "error" | null;
|
||||
downloaded_bytes: number;
|
||||
// null when the total isn't known yet.
|
||||
expected_bytes?: number | null;
|
||||
error?: string | null;
|
||||
}
|
||||
|
||||
export interface VideoLoadRequest {
|
||||
model_path: string;
|
||||
// Required for the gguf / single_file kinds, omitted for a full pipeline (a
|
||||
// diffusers repo loaded via from_pretrained).
|
||||
gguf_filename?: string;
|
||||
// How to load the model (omit to auto-detect from gguf_filename): "gguf" (single-file
|
||||
// GGUF transformer), "single_file" (single-file safetensors transformer), or "pipeline"
|
||||
// (a full diffusers repo). Non-GGUF kinds are restricted to unsloth/* or family bases.
|
||||
model_kind?: "gguf" | "single_file" | "pipeline";
|
||||
base_repo?: string;
|
||||
family_override?: string;
|
||||
hf_token?: string;
|
||||
// Advanced (load-time) tuning. All optional; omit for the backend's auto defaults.
|
||||
memory_mode?: "auto" | "fast" | "balanced" | "low_vram";
|
||||
speed_mode?: "off" | "eager" | "default" | "max";
|
||||
attention_backend?:
|
||||
| "auto"
|
||||
| "native"
|
||||
| "sdpa"
|
||||
| "cudnn"
|
||||
| "flash"
|
||||
| "flash2"
|
||||
| "flash3"
|
||||
| "flash4"
|
||||
| "sage"
|
||||
| "xformers"
|
||||
| "aiter";
|
||||
transformer_cache?: "off" | "fbcache";
|
||||
transformer_cache_threshold?: number;
|
||||
}
|
||||
|
||||
export interface VideoGenerateRequest {
|
||||
prompt: string;
|
||||
negative_prompt?: string;
|
||||
// Width/height/num_frames/fps default per loaded family (the backend snaps them to the
|
||||
// family's required multiples/lattice), so they are optional here.
|
||||
width?: number;
|
||||
height?: number;
|
||||
num_frames?: number;
|
||||
fps?: number;
|
||||
steps?: number;
|
||||
guidance?: number;
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
// A persisted clip's full generation recipe (the JSON sidecar of the MP4).
|
||||
export interface GalleryVideo {
|
||||
id: string;
|
||||
// Relative URL to fetch the MP4 bytes (auth-protected).
|
||||
url: string;
|
||||
prompt: string;
|
||||
negative_prompt?: string | null;
|
||||
width: number;
|
||||
height: number;
|
||||
num_frames: number;
|
||||
fps: number;
|
||||
duration_s: number;
|
||||
steps: number;
|
||||
guidance: number;
|
||||
seed: number;
|
||||
has_audio: boolean;
|
||||
model?: string | null;
|
||||
// Creation time (ISO 8601 timestamp).
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface VideoGenerateResponse {
|
||||
video: GalleryVideo;
|
||||
}
|
||||
|
||||
async function parseJson<T>(response: Response): Promise<T> {
|
||||
if (!response.ok) {
|
||||
throw new Error(await readFastApiError(response));
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
|
||||
export async function getVideoStatus(): Promise<VideoStatus> {
|
||||
return parseJson(await authFetch("/api/inference/video/status"));
|
||||
}
|
||||
|
||||
export async function getVideoLoadProgress(): Promise<VideoLoadProgress> {
|
||||
return parseJson(await authFetch("/api/inference/video/load-progress"));
|
||||
}
|
||||
|
||||
export async function getVideoGenerateProgress(): Promise<VideoGenerateProgress> {
|
||||
return parseJson(await authFetch("/api/inference/video/generate-progress"));
|
||||
}
|
||||
|
||||
export async function loadVideoModel(body: VideoLoadRequest): Promise<VideoStatus> {
|
||||
return parseJson(
|
||||
await authFetch("/api/inference/video/load", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
export async function generateVideo(
|
||||
body: VideoGenerateRequest,
|
||||
): Promise<VideoGenerateResponse> {
|
||||
return parseJson(
|
||||
await authFetch("/api/inference/video/generate", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** Request a cancel of the in-flight generation. Best-effort: the backend stops at the
|
||||
* next step boundary and raises the cancelled sentinel, which the caller maps to a 409. */
|
||||
export async function cancelVideoGeneration(): Promise<{ cancelled: boolean }> {
|
||||
return parseJson(
|
||||
await authFetch("/api/inference/video/generate/cancel", { method: "POST" }),
|
||||
);
|
||||
}
|
||||
|
||||
export async function unloadVideoModel(): Promise<VideoStatus> {
|
||||
return parseJson(await authFetch("/api/inference/video/unload", { method: "POST" }));
|
||||
}
|
||||
|
||||
export interface VideoGalleryPage {
|
||||
videos: GalleryVideo[];
|
||||
has_more: boolean;
|
||||
}
|
||||
|
||||
export async function getVideoGallery(offset = 0, limit = 50): Promise<VideoGalleryPage> {
|
||||
return parseJson(
|
||||
await authFetch(`/api/inference/video/gallery?offset=${offset}&limit=${limit}`),
|
||||
);
|
||||
}
|
||||
|
||||
export async function deleteGalleryVideo(id: string): Promise<void> {
|
||||
const res = await authFetch(`/api/inference/video/gallery/${id}`, { method: "DELETE" });
|
||||
if (!res.ok) throw new Error(await readFastApiError(res));
|
||||
}
|
||||
|
||||
export async function clearVideoGallery(): Promise<void> {
|
||||
const res = await authFetch("/api/inference/video/gallery", { method: "DELETE" });
|
||||
if (!res.ok) throw new Error(await readFastApiError(res));
|
||||
}
|
||||
|
||||
/** Fetch a gallery MP4 (auth-protected, so it can't be a plain <video src>) and wrap it
|
||||
* in an object URL. Callers must revoke the URL when done. Mirrors the images gallery. */
|
||||
export async function fetchGalleryVideoObjectUrl(url: string): Promise<string> {
|
||||
const res = await authFetch(url);
|
||||
if (!res.ok) throw new Error(await readFastApiError(res));
|
||||
return URL.createObjectURL(await res.blob());
|
||||
}
|
||||
4
studio/frontend/src/features/video/index.ts
Normal file
4
studio/frontend/src/features/video/index.ts
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
export { VideoPage } from "./video-page";
|
||||
1475
studio/frontend/src/features/video/video-page.tsx
Normal file
1475
studio/frontend/src/features/video/video-page.tsx
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -40,6 +40,7 @@ export const en = {
|
|||
train: "Train",
|
||||
recipes: "Recipes",
|
||||
images: "Images",
|
||||
video: "Video",
|
||||
export: "Export",
|
||||
recents: "Recents",
|
||||
settings: "Settings",
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ export const zhCN = {
|
|||
train: "训练",
|
||||
recipes: "配方",
|
||||
images: "图像",
|
||||
video: "视频",
|
||||
export: "导出",
|
||||
recents: "最近",
|
||||
settings: "设置",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue