From d4c2fc526a899863e28983dcb1ba24dae533ddee Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 18 Jul 2026 07:38:10 +0000 Subject: [PATCH] Wire the hosted LTX-2 and LTX-2.3 int8/fp8 checkpoints The ltx-2 family resolves unsloth/LTX-2-FP8 for the base pipeline and unsloth/LTX-2.3-FP8 for the 2.3 distilled weights via the variant table (keyed on the lowercased 2.3 base; a checkpoint baked from the base DiT fails base_model_id validation against 2.3). The 2.3 checkpoints are built from the official single file through the exact runtime assembly path: the builder gains --ltx23-single-file and load_ltx23_pipeline gains transformer_override so a pre-built DiT can be injected while the single file contributes only connectors, VAEs and vocoder. Every checkpoint is gate-validated through the real VideoBackend and verified bit-identical to on-the-fly quantization (int8: 4186 tensors, 1350 quantized, 0 mismatches; fp8: 4186 tensors, 1360 quantized, 0 mismatches), with same-seed generation deterministic per load. --- scripts/build_prequant_checkpoint.py | 54 +++++++++++++++++-- .../backend/core/inference/video_families.py | 15 ++++++ studio/backend/core/inference/video_ltx2.py | 25 ++++++--- studio/backend/tests/test_video_families.py | 28 ++++++++-- 4 files changed, 106 insertions(+), 16 deletions(-) diff --git a/scripts/build_prequant_checkpoint.py b/scripts/build_prequant_checkpoint.py index bf0b5e919a..b4b9f46ac5 100644 --- a/scripts/build_prequant_checkpoint.py +++ b/scripts/build_prequant_checkpoint.py @@ -45,6 +45,14 @@ def main(argv = None) -> int: p.add_argument("--min-features", type = int, default = 512) p.add_argument("--dtype", default = "bfloat16", choices = ["bfloat16"]) p.add_argument("--hf-token", default = None) + p.add_argument( + "--ltx23-single-file", + default = None, + help = "repo_id:filename of an LTX-2.3 single-file checkpoint; the transformer is " + "assembled via the runtime's load_ltx23_transformer (2.3 key renames + config " + "overrides on the --base LTX-2 config) instead of from_pretrained, and " + "base_model_id records the single-file repo (the 2.3 weights identity).", + ) p.add_argument( "--upload-repo", default = None, help = "optional HF repo id to upload the checkpoint to" ) @@ -89,11 +97,42 @@ def main(argv = None) -> int: transformer_cls = getattr(diffusers, fam.transformer_class) print(f"== build prequant ({fam.name}/{scheme}, min_feat={args.min_features}) ==", flush = True) - print(f" loading dense transformer from {args.base} (subfolder={args.subfolder}) ...", flush = True) t0 = time.time() - transformer = transformer_cls.from_pretrained( - args.base, subfolder = args.subfolder, torch_dtype = torch.bfloat16, token = args.hf_token - ).to("cuda") + base_model_id = args.base + if args.ltx23_single_file: + # LTX-2.3 ships one .safetensors carrying DiT + connectors + VAEs; the runtime + # assembles the transformer via load_ltx23_transformer (2.3-only key renames + + # config overrides merged into the LTX-2 base config). Reuse that EXACT path so + # offline == runtime, and stamp the single-file repo as the weights identity. + from huggingface_hub import hf_hub_download + + from core.inference.video_ltx2 import _split_checkpoint, load_ltx23_transformer + + sf_repo, sf_name = args.ltx23_single_file.split(":", 1) + print(f" loading 2.3 single file {sf_repo}:{sf_name} ...", flush = True) + local = hf_hub_download(sf_repo, sf_name, token = args.hf_token) + from diffusers.loaders.single_file_utils import load_single_file_checkpoint + + state = load_single_file_checkpoint(str(local)) + groups = _split_checkpoint(state) + del state + transformer = load_ltx23_transformer( + groups["dit"], + base_repo = args.base, + torch_dtype = torch.bfloat16, + is_gguf = False, + hf_token = args.hf_token, + ).to("cuda") + del groups + base_model_id = sf_repo + else: + print( + f" loading dense transformer from {args.base} (subfolder={args.subfolder}) ...", + flush = True, + ) + transformer = transformer_cls.from_pretrained( + args.base, subfolder = args.subfolder, torch_dtype = torch.bfloat16, token = args.hf_token + ).to("cuda") print(f" quantising in place ({scheme}) ...", flush = True) # Mirror the runtime path EXACTLY (offline == runtime, LPIPS-0 invariant): int8 skips the # M=1 modulation / conditioning-embedder projections (else the checkpoint bakes them int8 and @@ -123,7 +162,7 @@ def main(argv = None) -> int: for k, v in transformer.state_dict().items() } metadata = { - "base_model_id": args.base, + "base_model_id": base_model_id, "family": fam.name, "scheme": scheme, "min_features": args.min_features, @@ -143,6 +182,11 @@ def main(argv = None) -> int: # (the runtime now requires per-row; see FP8_GRANULARITY). if scheme == TQ_FP8: metadata["fp8_granularity"] = FP8_GRANULARITY + if args.ltx23_single_file: + # The 2.3 transformer config is the LTX-2 base config plus the 2.3 overrides; no + # diffusers repo carries it as a subfolder, so bake the merged dict for a future + # meta-init (the current loader path receives the module via transformer_override). + metadata["transformer_config"] = dict(transformer.config) ckpt = { "format": PREQUANT_FORMAT, "metadata": metadata, diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py index 2a1c761bd3..7a71c694ca 100644 --- a/studio/backend/core/inference/video_families.py +++ b/studio/backend/core/inference/video_families.py @@ -122,6 +122,21 @@ _FAMILIES: tuple[VideoFamily, ...] = ( # Pre-cast Gemma3-12B TE (hub store is fp32 ~49 GB, pre-cast ~13.2 GB): the biggest # download win of the hosted TE set. te_prequant_repos = (("fp8", "text_encoder", "unsloth/LTX-2-FP8"),), + # Gate-validated hosted checkpoints for the LTX-2 BASE pipeline weights. + prequant_repos = ( + ("int8", "unsloth/LTX-2-FP8"), + ("fp8", "unsloth/LTX-2-FP8"), + ), + # The 2.3 distilled checkpoints are baked from the Lightricks/LTX-2.3 single-file + # weights (different DiT than the base). The loader cannot consume them yet -- 2.3 + # loads as single_file/GGUF kind, where the pipeline-kind prequant shortcut never + # runs -- but the table keeps resolution correct (family_prequant_repo with the 2.3 + # base returns the 2.3 repo, whose base_model_id validation would refuse the base + # LTX-2 checkpoint) for when that path learns transformer_quant. + prequant_variant_repos = ( + ("lightricks/ltx-2.3", "int8", "unsloth/LTX-2.3-FP8"), + ("lightricks/ltx-2.3", "fp8", "unsloth/LTX-2.3-FP8"), + ), ), # Wan2.2-TI2V-5B (diffusers >= 0.35, verified on 0.39): ~5B single-stream video DiT (UMT5 # text encoder). No audio, no second expert (boundary_ratio null, transformer_2 null), so diff --git a/studio/backend/core/inference/video_ltx2.py b/studio/backend/core/inference/video_ltx2.py index 430e003aad..c91bd80518 100644 --- a/studio/backend/core/inference/video_ltx2.py +++ b/studio/backend/core/inference/video_ltx2.py @@ -485,10 +485,15 @@ def load_ltx23_pipeline( torch_dtype: Any, is_gguf: bool, hf_token: Optional[str] = None, + transformer_override: Any = None, ) -> Any: """Full LTX-2.3 pipeline from a single-file/GGUF checkpoint. Assembled per-component (constructor, not from_pretrained) because the base model_index pins LTX2Vocoder while 2.3 - needs LTX2VocoderWithBWE, which the type gate would reject.""" + needs LTX2VocoderWithBWE, which the type gate would reject. + + ``transformer_override`` supplies a pre-built DiT (e.g. a pre-quantized checkpoint); the + single file then contributes only the connectors / VAEs / vocoder groups and its DiT + tensors are dropped unread.""" import transformers from diffusers import LTX2Pipeline from diffusers.loaders.single_file_utils import load_single_file_checkpoint @@ -514,13 +519,17 @@ def load_ltx23_pipeline( "instead (Q8_0 for the highest fidelity) or the official bf16 checkpoint." ) - transformer = load_ltx23_transformer( - groups["dit"], - base_repo = base_repo, - torch_dtype = torch_dtype, - is_gguf = is_gguf, - hf_token = hf_token, - ) + if transformer_override is not None: + transformer = transformer_override + groups.pop("dit", None) + else: + transformer = load_ltx23_transformer( + groups["dit"], + base_repo = base_repo, + torch_dtype = torch_dtype, + is_gguf = is_gguf, + hf_token = hf_token, + ) connectors = load_ltx23_connectors( groups["connectors"], variant = variant, diff --git a/studio/backend/tests/test_video_families.py b/studio/backend/tests/test_video_families.py index 47d0fae6aa..b04a80785d 100644 --- a/studio/backend/tests/test_video_families.py +++ b/studio/backend/tests/test_video_families.py @@ -358,9 +358,17 @@ def test_video_prequant_repo_wiring(): hv720 = detect_video_family("hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v") assert family_prequant_repo(hv720, "int8") == "unsloth/HunyuanVideo-1.5-720p-FP8" assert family_prequant_repo(hv720, "int8") != family_prequant_repo(hv480, "int8") - # LTX has no measured quant recipe -> deliberately unwired. - ltx = detect_video_family("unsloth/LTX-2.3-GGUF") - assert family_prequant_repo(ltx, "int8") is None + # LTX-2 base carries int8 + fp8 (both gate-validated); the 2.3 distilled weights get + # their OWN repo via the variant table, keyed on the lowercased 2.3 base, because a + # checkpoint baked from the base LTX-2 DiT fails base_model_id validation against 2.3. + ltx = detect_video_family("Lightricks/LTX-2") + assert family_prequant_repo(ltx, "int8") == "unsloth/LTX-2-FP8" + assert family_prequant_repo(ltx, "fp8") == "unsloth/LTX-2-FP8" + assert family_prequant_repo(ltx, "fp8", "Lightricks/LTX-2.3") == "unsloth/LTX-2.3-FP8" + assert family_prequant_repo(ltx, "int8", "Lightricks/LTX-2.3") == "unsloth/LTX-2.3-FP8" + # An unknown LTX variant falls back to the family default (the loader's base_model_id + # check then refuses a mismatched checkpoint and dense-quantises). + assert family_prequant_repo(ltx, "fp8", "someone/ltx-finetune") == "unsloth/LTX-2-FP8" def test_video_prequant_dual_expert_resolution(): @@ -375,3 +383,17 @@ def test_video_prequant_dual_expert_resolution(): assert first.filename == "Wan2.2-T2V-A14B-INT8.pt" assert second.filename == "Wan2.2-T2V-A14B-INT8-2.pt" assert second.fallback_filename == "transformer_2_int8.pt" + + +def test_video_prequant_ltx_variant_resolution(): + # The 2.3 base selects the 2.3 repo (its checkpoints stamp base_model_id + # Lightricks/LTX-2.3); without a base the family default (base LTX-2 weights) resolves. + from core.inference.diffusion_prequant import resolve_prequant_source + + fam = detect_video_family("Lightricks/LTX-2") + base = resolve_prequant_source(fam, "fp8") + assert base.location == "unsloth/LTX-2-FP8" + assert base.filename == "LTX-2-FP8.pt" + v23 = resolve_prequant_source(fam, "int8", base_repo = "Lightricks/LTX-2.3") + assert v23.location == "unsloth/LTX-2.3-FP8" + assert v23.filename == "LTX-2.3-INT8.pt"