Add Video tab to Studio frontend

Add a Video generation page that mirrors the Images feature's create
workflow. It loads a text-to-video model, generates a clip, and plays it
back inline with the gallery of past clips.

- src/features/video/api.ts: typed client for the /api/inference/video
  routes (load, load-progress, generate, generate-progress, cancel,
  status, unload, gallery CRUD, and an auth-protected MP4 blob fetch).
- src/features/video/video-page.tsx: the page. Curated model picker
  (LTX 2.3 distilled GGUF, LTX 2 base pipeline), prompt and negative
  prompt, resolution preset select, duration select over the family's
  temporal lattice, fixed fps display, steps and guidance sliders seeded
  from per-model defaults, seed box. Generate polls per-step progress
  with a phase label and ETA and a Cancel button, then plays the result
  in a video player with a download button and an audio badge. Gallery
  strip below with per-card delete and clear all. Right-docked Advanced
  panel for memory, speed, attention, and step-cache with Auto badges
  fed from the resolved status.
- Register the page: router child, /video route, sidebar nav item with a
  video icon after Images, and the __root keep-alive mount so an
  in-flight generation survives leaving the tab.
- Add the text-to-video task to the model picker so video models never
  appear in the chat picker.
- Add the video nav label to the en and zh-CN locales.
This commit is contained in:
Daniel Han 2026-07-04 13:30:42 +00:00
commit a487f3c1c0
10 changed files with 1789 additions and 9 deletions

View file

@ -11,6 +11,7 @@ import { Route as dataRecipeRoute } from "./routes/data-recipes.$recipeId";
import { Route as chatRoute } from "./routes/chat";
import { Route as exportRoute } from "./routes/export";
import { Route as imagesRoute } from "./routes/images";
import { Route as videoRoute } from "./routes/video";
import { Route as indexRoute } from "./routes/index";
import { Route as loginRoute } from "./routes/login";
import { Route as hubRoute } from "./routes/hub";
@ -32,6 +33,7 @@ const routeTree = rootRoute.addChildren([
projectsRoute,
exportRoute,
imagesRoute,
videoRoute,
dataRecipesRoute,
dataRecipeRoute,
]);

View file

@ -65,6 +65,13 @@ const ImagesPage = lazy(() =>
import("@/features/images").then((m) => ({ default: m.ImagesPage })),
);
// VideoPage gets the same persistent-mount treatment as ImagesPage so an in-flight
// generation survives leaving the tab. Kept lazy so its bundle loads only on the first
// /video visit.
const VideoPage = lazy(() =>
import("@/features/video").then((m) => ({ default: m.VideoPage })),
);
function PersonalizationSyncMount() {
usePersonalizationSync(hasAuthToken());
return null;
@ -167,13 +174,24 @@ function RootLayout() {
setImagesMounted(true);
}
const shouldMountImages = isImagesRoute || imagesMounted;
// Chat and Images both render their own full-height shell (a fixed top rail + an
// internally-scrolling body), so both want the chat-style layout: no outer pt-14
// inset and no outer scroll. Keying the layout off isChatRoute alone gave /images
// the non-chat pt-14 + outer overflow, pushing its picker down and clipping the
// Same persistent-mount treatment for /video so a long generation keeps running when
// the user flips to another tab (VideoPage reads no URL search, so it needs no freeze
// dance -- just the mount latch). Mounts lazily on first /video visit, then stays
// mounted, hidden+inert while off-route.
const isVideoRoute = pathname === "/video";
const [videoMounted, setVideoMounted] = useState(isVideoRoute);
if (isVideoRoute && !videoMounted) {
setVideoMounted(true);
}
const shouldMountVideo = isVideoRoute || videoMounted;
// Chat, Images and Video all render their own full-height shell (a fixed top rail + an
// internally-scrolling body), so all three want the chat-style layout: no outer pt-14
// inset and no outer scroll. Keying the layout off isChatRoute alone gave /images and
// /video the non-chat pt-14 + outer overflow, pushing the picker down and clipping the
// bottom gallery. Treat them the same for the container padding/overflow only; the
// keep-alive mounts below stay keyed to each specific route.
const isChatLike = isChatRoute || isImagesRoute;
const isChatLike = isChatRoute || isImagesRoute || isVideoRoute;
useTrainingUnloadGuard();
// Global export driver: streams worker logs and tracks status from any route
@ -305,12 +323,29 @@ function RootLayout() {
</Suspense>
</div>
)}
{/* Same keep-alive treatment for Video so a long generation keeps running
off-tab; `active` force-closes its body-portaled overlays (model selector,
recipe popover) so none can bleed over another tab while hidden. */}
{shouldMountVideo && (
<div
className={
isVideoRoute
? "flex min-h-0 min-w-0 flex-1 basis-0 flex-col overflow-hidden"
: "hidden"
}
inert={!isVideoRoute || undefined}
>
<Suspense fallback={<RouteFallback />}>
<VideoPage active={isVideoRoute} />
</Suspense>
</div>
)}
{/* Use mode="popLayout" instead of "wait" to prevent UI freezes when
switching from heavy pages (like Export with many checkpoints).
"popLayout" allows the new route to mount immediately while the
old one animates out, avoiding blocking on expensive exit renders.
See issue #5850. */}
{!isChatRoute && !isImagesRoute && (
{!isChatRoute && !isImagesRoute && !isVideoRoute && (
<AnimatePresence initial={false} mode="popLayout">
<motion.div
key={pathname}

View file

@ -0,0 +1,16 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { createRoute } from "@tanstack/react-router";
import { requireAuth } from "../auth-guards";
import { Route as rootRoute } from "./__root";
// RootLayout renders VideoPage persistently (so an in-flight generation is not cancelled
// when leaving the tab); this route only owns the URL + auth gate.
export const Route = createRoute({
getParentRoute: () => rootRoute,
path: "/video",
staticData: { title: "Video" },
beforeLoad: () => requireAuth(),
component: () => null,
});

View file

@ -80,6 +80,7 @@ import {
Settings02Icon,
Sun03Icon,
TestTube01Icon,
Video01Icon,
ZapIcon,
} from "@hugeicons/core-free-icons";
import {
@ -1207,6 +1208,15 @@ export function AppSidebar() {
closeMobileIfOpen();
}}
/>
<NavItem
icon={Video01Icon}
label={t("shell.navigation.video")}
active={pathname === "/video" || pathname.startsWith("/video/")}
onClick={() => {
navigate({ to: "/video" });
closeMobileIfOpen();
}}
/>
{/* Train has a labelled section when expanded; plain icon here only when collapsed. */}
<NavItem
icon={TestTubeOutlineIcon}

View file

@ -1053,15 +1053,25 @@ export const IMAGE_GEN_TASKS = [
"image-text-to-image",
] as const;
// Video-generation pipeline tasks: handled by the Video page, never loadable as
// chat models. The backend reports "text-to-video" for video-diffusion GGUFs. The
// Video page reuses this as its picker's `task` filter, so it lives here.
export const VIDEO_GEN_TASKS = ["text-to-video"] as const;
// Diffusion GGUF archs the Images backend can't assemble yet (SD/SDXL/PixArt/Wan/
// ...). The backend tags them with this task so the chat picker hides them -- they
// die with "unknown model architecture" in llama.cpp -- while the Images picker,
// which filters on IMAGE_GEN_TASKS, also leaves them out (they'd 400 on load).
const UNSUPPORTED_DIFFUSION_TASK = "image-diffusion-unsupported";
// Tasks that must never appear as a loadable chat model: the Images-handled
// generation tasks plus the non-loadable diffusion tag above.
const NON_CHAT_TASKS: readonly string[] = [...IMAGE_GEN_TASKS, UNSUPPORTED_DIFFUSION_TASK];
// Tasks that must never appear as a loadable chat model: the Images- and Video-handled
// generation tasks plus the non-loadable diffusion tag above. Keeping text-to-video here
// stops a downloaded video GGUF from showing up as a loadable chat model (it would 400).
const NON_CHAT_TASKS: readonly string[] = [
...IMAGE_GEN_TASKS,
...VIDEO_GEN_TASKS,
UNSUPPORTED_DIFFUSION_TASK,
];
// Editing/inpaint checkpoints are tagged image-to-image but need an input image,
// which the text-to-image backend rejects (mirrors its _EDIT_KEYWORDS). Hidden by

View file

@ -0,0 +1,226 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import { authFetch } from "@/features/auth";
import { readFastApiError } from "@/lib/format-fastapi-error";
// One Advanced control's resolved value + provenance, for the "Auto: X" badges. Same
// shape the diffusion status uses. `value` is the engaged value (a scheme/mode string,
// null when off, or a boolean); `source` is "auto" (this backend decided) or "explicit"
// (the caller set it); `reason` is the short why shown as a tooltip.
export interface VideoResolvedControl {
value: string | boolean | null;
source: "auto" | "explicit";
reason: string;
}
// Per-family generation defaults + shape constraints, from status.defaults when loaded.
export interface VideoGenerationDefaults {
steps: number;
guidance: number;
num_frames: number;
fps: number;
// Temporal lattice: valid frame counts are k * frame_step + 1.
frame_step: number;
// Width/height must be divisible by this.
resolution_multiple: number;
// (width, height) presets the UI offers, default first.
resolution_presets: Array<[number, number]>;
}
export interface VideoStatus {
loaded: boolean;
repo_id: string | null;
family: string | null;
base_repo: string | null;
device: string | null;
dtype: string | null;
// Resolved load kind: "gguf" | "single_file" | "pipeline". Null when not loaded.
model_kind?: string | null;
// Resolved offload policy: none | group | model | sequential.
offload_policy?: string | null;
vae_tiling: boolean;
memory_mode?: string | null;
speed_mode?: string | null;
// Speed optimisations actually engaged.
speed_optims: string[];
attention_backend?: string | null;
transformer_cache?: string | null;
// Whether the loaded family produces a synchronized audio track.
has_audio: boolean;
// Per-family generation defaults + shape constraints; null when unloaded.
defaults?: VideoGenerationDefaults | null;
// Per-Advanced-control provenance, keyed by control name (memory_mode, speed_mode,
// attention_backend, transformer_cache). The "Auto: X" badges read it. Null when
// nothing is loaded or on a backend that doesn't record it.
resolved?: Record<string, VideoResolvedControl> | null;
}
export interface VideoGenerateProgress {
active: boolean;
// "denoise" | "export" | null.
phase?: string | null;
step: number;
total: number;
eta_seconds?: number | null;
}
export interface VideoLoadProgress {
phase: "downloading" | "finalizing" | "ready" | "error" | null;
downloaded_bytes: number;
// null when the total isn't known yet.
expected_bytes?: number | null;
error?: string | null;
}
export interface VideoLoadRequest {
model_path: string;
// Required for the gguf / single_file kinds, omitted for a full pipeline (a
// diffusers repo loaded via from_pretrained).
gguf_filename?: string;
// How to load the model (omit to auto-detect from gguf_filename): "gguf" (single-file
// GGUF transformer), "single_file" (single-file safetensors transformer), or "pipeline"
// (a full diffusers repo). Non-GGUF kinds are restricted to unsloth/* or family bases.
model_kind?: "gguf" | "single_file" | "pipeline";
base_repo?: string;
family_override?: string;
hf_token?: string;
// Advanced (load-time) tuning. All optional; omit for the backend's auto defaults.
memory_mode?: "auto" | "fast" | "balanced" | "low_vram";
speed_mode?: "off" | "eager" | "default" | "max";
attention_backend?:
| "auto"
| "native"
| "sdpa"
| "cudnn"
| "flash"
| "flash2"
| "flash3"
| "flash4"
| "sage"
| "xformers"
| "aiter";
transformer_cache?: "off" | "fbcache";
transformer_cache_threshold?: number;
}
export interface VideoGenerateRequest {
prompt: string;
negative_prompt?: string;
// Width/height/num_frames/fps default per loaded family (the backend snaps them to the
// family's required multiples/lattice), so they are optional here.
width?: number;
height?: number;
num_frames?: number;
fps?: number;
steps?: number;
guidance?: number;
seed?: number;
}
// A persisted clip's full generation recipe (the JSON sidecar of the MP4).
export interface GalleryVideo {
id: string;
// Relative URL to fetch the MP4 bytes (auth-protected).
url: string;
prompt: string;
negative_prompt?: string | null;
width: number;
height: number;
num_frames: number;
fps: number;
duration_s: number;
steps: number;
guidance: number;
seed: number;
has_audio: boolean;
model?: string | null;
// Creation time (ISO 8601 timestamp).
created_at: string;
}
export interface VideoGenerateResponse {
video: GalleryVideo;
}
async function parseJson<T>(response: Response): Promise<T> {
if (!response.ok) {
throw new Error(await readFastApiError(response));
}
return (await response.json()) as T;
}
export async function getVideoStatus(): Promise<VideoStatus> {
return parseJson(await authFetch("/api/inference/video/status"));
}
export async function getVideoLoadProgress(): Promise<VideoLoadProgress> {
return parseJson(await authFetch("/api/inference/video/load-progress"));
}
export async function getVideoGenerateProgress(): Promise<VideoGenerateProgress> {
return parseJson(await authFetch("/api/inference/video/generate-progress"));
}
export async function loadVideoModel(body: VideoLoadRequest): Promise<VideoStatus> {
return parseJson(
await authFetch("/api/inference/video/load", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
}),
);
}
export async function generateVideo(
body: VideoGenerateRequest,
): Promise<VideoGenerateResponse> {
return parseJson(
await authFetch("/api/inference/video/generate", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
}),
);
}
/** Request a cancel of the in-flight generation. Best-effort: the backend stops at the
* next step boundary and raises the cancelled sentinel, which the caller maps to a 409. */
export async function cancelVideoGeneration(): Promise<{ cancelled: boolean }> {
return parseJson(
await authFetch("/api/inference/video/generate/cancel", { method: "POST" }),
);
}
export async function unloadVideoModel(): Promise<VideoStatus> {
return parseJson(await authFetch("/api/inference/video/unload", { method: "POST" }));
}
export interface VideoGalleryPage {
videos: GalleryVideo[];
has_more: boolean;
}
export async function getVideoGallery(offset = 0, limit = 50): Promise<VideoGalleryPage> {
return parseJson(
await authFetch(`/api/inference/video/gallery?offset=${offset}&limit=${limit}`),
);
}
export async function deleteGalleryVideo(id: string): Promise<void> {
const res = await authFetch(`/api/inference/video/gallery/${id}`, { method: "DELETE" });
if (!res.ok) throw new Error(await readFastApiError(res));
}
export async function clearVideoGallery(): Promise<void> {
const res = await authFetch("/api/inference/video/gallery", { method: "DELETE" });
if (!res.ok) throw new Error(await readFastApiError(res));
}
/** Fetch a gallery MP4 (auth-protected, so it can't be a plain <video src>) and wrap it
* in an object URL. Callers must revoke the URL when done. Mirrors the images gallery. */
export async function fetchGalleryVideoObjectUrl(url: string): Promise<string> {
const res = await authFetch(url);
if (!res.ok) throw new Error(await readFastApiError(res));
return URL.createObjectURL(await res.blob());
}

View file

@ -0,0 +1,4 @@
// SPDX-License-Identifier: AGPL-3.0-only
// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
export { VideoPage } from "./video-page";

File diff suppressed because it is too large Load diff

View file

@ -40,6 +40,7 @@ export const en = {
train: "Train",
recipes: "Recipes",
images: "Images",
video: "Video",
export: "Export",
recents: "Recents",
settings: "Settings",

View file

@ -40,6 +40,7 @@ export const zhCN = {
train: "训练",
recipes: "配方",
images: "图像",
video: "视频",
export: "导出",
recents: "最近",
settings: "设置",