Wire the hosted LTX-2 and LTX-2.3 int8/fp8 checkpoints
The ltx-2 family resolves unsloth/LTX-2-FP8 for the base pipeline and unsloth/LTX-2.3-FP8 for the 2.3 distilled weights via the variant table (keyed on the lowercased 2.3 base; a checkpoint baked from the base DiT fails base_model_id validation against 2.3). The 2.3 checkpoints are built from the official single file through the exact runtime assembly path: the builder gains --ltx23-single-file and load_ltx23_pipeline gains transformer_override so a pre-built DiT can be injected while the single file contributes only connectors, VAEs and vocoder. Every checkpoint is gate-validated through the real VideoBackend and verified bit-identical to on-the-fly quantization (int8: 4186 tensors, 1350 quantized, 0 mismatches; fp8: 4186 tensors, 1360 quantized, 0 mismatches), with same-seed generation deterministic per load.
This commit is contained in:
parent
a3942d9924
commit
d4c2fc526a
4 changed files with 106 additions and 16 deletions
|
|
@ -45,6 +45,14 @@ def main(argv = None) -> int:
|
|||
p.add_argument("--min-features", type = int, default = 512)
|
||||
p.add_argument("--dtype", default = "bfloat16", choices = ["bfloat16"])
|
||||
p.add_argument("--hf-token", default = None)
|
||||
p.add_argument(
|
||||
"--ltx23-single-file",
|
||||
default = None,
|
||||
help = "repo_id:filename of an LTX-2.3 single-file checkpoint; the transformer is "
|
||||
"assembled via the runtime's load_ltx23_transformer (2.3 key renames + config "
|
||||
"overrides on the --base LTX-2 config) instead of from_pretrained, and "
|
||||
"base_model_id records the single-file repo (the 2.3 weights identity).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--upload-repo", default = None, help = "optional HF repo id to upload the checkpoint to"
|
||||
)
|
||||
|
|
@ -89,11 +97,42 @@ def main(argv = None) -> int:
|
|||
transformer_cls = getattr(diffusers, fam.transformer_class)
|
||||
|
||||
print(f"== build prequant ({fam.name}/{scheme}, min_feat={args.min_features}) ==", flush = True)
|
||||
print(f" loading dense transformer from {args.base} (subfolder={args.subfolder}) ...", flush = True)
|
||||
t0 = time.time()
|
||||
transformer = transformer_cls.from_pretrained(
|
||||
args.base, subfolder = args.subfolder, torch_dtype = torch.bfloat16, token = args.hf_token
|
||||
).to("cuda")
|
||||
base_model_id = args.base
|
||||
if args.ltx23_single_file:
|
||||
# LTX-2.3 ships one .safetensors carrying DiT + connectors + VAEs; the runtime
|
||||
# assembles the transformer via load_ltx23_transformer (2.3-only key renames +
|
||||
# config overrides merged into the LTX-2 base config). Reuse that EXACT path so
|
||||
# offline == runtime, and stamp the single-file repo as the weights identity.
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
from core.inference.video_ltx2 import _split_checkpoint, load_ltx23_transformer
|
||||
|
||||
sf_repo, sf_name = args.ltx23_single_file.split(":", 1)
|
||||
print(f" loading 2.3 single file {sf_repo}:{sf_name} ...", flush = True)
|
||||
local = hf_hub_download(sf_repo, sf_name, token = args.hf_token)
|
||||
from diffusers.loaders.single_file_utils import load_single_file_checkpoint
|
||||
|
||||
state = load_single_file_checkpoint(str(local))
|
||||
groups = _split_checkpoint(state)
|
||||
del state
|
||||
transformer = load_ltx23_transformer(
|
||||
groups["dit"],
|
||||
base_repo = args.base,
|
||||
torch_dtype = torch.bfloat16,
|
||||
is_gguf = False,
|
||||
hf_token = args.hf_token,
|
||||
).to("cuda")
|
||||
del groups
|
||||
base_model_id = sf_repo
|
||||
else:
|
||||
print(
|
||||
f" loading dense transformer from {args.base} (subfolder={args.subfolder}) ...",
|
||||
flush = True,
|
||||
)
|
||||
transformer = transformer_cls.from_pretrained(
|
||||
args.base, subfolder = args.subfolder, torch_dtype = torch.bfloat16, token = args.hf_token
|
||||
).to("cuda")
|
||||
print(f" quantising in place ({scheme}) ...", flush = True)
|
||||
# Mirror the runtime path EXACTLY (offline == runtime, LPIPS-0 invariant): int8 skips the
|
||||
# M=1 modulation / conditioning-embedder projections (else the checkpoint bakes them int8 and
|
||||
|
|
@ -123,7 +162,7 @@ def main(argv = None) -> int:
|
|||
for k, v in transformer.state_dict().items()
|
||||
}
|
||||
metadata = {
|
||||
"base_model_id": args.base,
|
||||
"base_model_id": base_model_id,
|
||||
"family": fam.name,
|
||||
"scheme": scheme,
|
||||
"min_features": args.min_features,
|
||||
|
|
@ -143,6 +182,11 @@ def main(argv = None) -> int:
|
|||
# (the runtime now requires per-row; see FP8_GRANULARITY).
|
||||
if scheme == TQ_FP8:
|
||||
metadata["fp8_granularity"] = FP8_GRANULARITY
|
||||
if args.ltx23_single_file:
|
||||
# The 2.3 transformer config is the LTX-2 base config plus the 2.3 overrides; no
|
||||
# diffusers repo carries it as a subfolder, so bake the merged dict for a future
|
||||
# meta-init (the current loader path receives the module via transformer_override).
|
||||
metadata["transformer_config"] = dict(transformer.config)
|
||||
ckpt = {
|
||||
"format": PREQUANT_FORMAT,
|
||||
"metadata": metadata,
|
||||
|
|
|
|||
|
|
@ -122,6 +122,21 @@ _FAMILIES: tuple[VideoFamily, ...] = (
|
|||
# Pre-cast Gemma3-12B TE (hub store is fp32 ~49 GB, pre-cast ~13.2 GB): the biggest
|
||||
# download win of the hosted TE set.
|
||||
te_prequant_repos = (("fp8", "text_encoder", "unsloth/LTX-2-FP8"),),
|
||||
# Gate-validated hosted checkpoints for the LTX-2 BASE pipeline weights.
|
||||
prequant_repos = (
|
||||
("int8", "unsloth/LTX-2-FP8"),
|
||||
("fp8", "unsloth/LTX-2-FP8"),
|
||||
),
|
||||
# The 2.3 distilled checkpoints are baked from the Lightricks/LTX-2.3 single-file
|
||||
# weights (different DiT than the base). The loader cannot consume them yet -- 2.3
|
||||
# loads as single_file/GGUF kind, where the pipeline-kind prequant shortcut never
|
||||
# runs -- but the table keeps resolution correct (family_prequant_repo with the 2.3
|
||||
# base returns the 2.3 repo, whose base_model_id validation would refuse the base
|
||||
# LTX-2 checkpoint) for when that path learns transformer_quant.
|
||||
prequant_variant_repos = (
|
||||
("lightricks/ltx-2.3", "int8", "unsloth/LTX-2.3-FP8"),
|
||||
("lightricks/ltx-2.3", "fp8", "unsloth/LTX-2.3-FP8"),
|
||||
),
|
||||
),
|
||||
# Wan2.2-TI2V-5B (diffusers >= 0.35, verified on 0.39): ~5B single-stream video DiT (UMT5
|
||||
# text encoder). No audio, no second expert (boundary_ratio null, transformer_2 null), so
|
||||
|
|
|
|||
|
|
@ -485,10 +485,15 @@ def load_ltx23_pipeline(
|
|||
torch_dtype: Any,
|
||||
is_gguf: bool,
|
||||
hf_token: Optional[str] = None,
|
||||
transformer_override: Any = None,
|
||||
) -> Any:
|
||||
"""Full LTX-2.3 pipeline from a single-file/GGUF checkpoint. Assembled per-component
|
||||
(constructor, not from_pretrained) because the base model_index pins LTX2Vocoder while 2.3
|
||||
needs LTX2VocoderWithBWE, which the type gate would reject."""
|
||||
needs LTX2VocoderWithBWE, which the type gate would reject.
|
||||
|
||||
``transformer_override`` supplies a pre-built DiT (e.g. a pre-quantized checkpoint); the
|
||||
single file then contributes only the connectors / VAEs / vocoder groups and its DiT
|
||||
tensors are dropped unread."""
|
||||
import transformers
|
||||
from diffusers import LTX2Pipeline
|
||||
from diffusers.loaders.single_file_utils import load_single_file_checkpoint
|
||||
|
|
@ -514,13 +519,17 @@ def load_ltx23_pipeline(
|
|||
"instead (Q8_0 for the highest fidelity) or the official bf16 checkpoint."
|
||||
)
|
||||
|
||||
transformer = load_ltx23_transformer(
|
||||
groups["dit"],
|
||||
base_repo = base_repo,
|
||||
torch_dtype = torch_dtype,
|
||||
is_gguf = is_gguf,
|
||||
hf_token = hf_token,
|
||||
)
|
||||
if transformer_override is not None:
|
||||
transformer = transformer_override
|
||||
groups.pop("dit", None)
|
||||
else:
|
||||
transformer = load_ltx23_transformer(
|
||||
groups["dit"],
|
||||
base_repo = base_repo,
|
||||
torch_dtype = torch_dtype,
|
||||
is_gguf = is_gguf,
|
||||
hf_token = hf_token,
|
||||
)
|
||||
connectors = load_ltx23_connectors(
|
||||
groups["connectors"],
|
||||
variant = variant,
|
||||
|
|
|
|||
|
|
@ -358,9 +358,17 @@ def test_video_prequant_repo_wiring():
|
|||
hv720 = detect_video_family("hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v")
|
||||
assert family_prequant_repo(hv720, "int8") == "unsloth/HunyuanVideo-1.5-720p-FP8"
|
||||
assert family_prequant_repo(hv720, "int8") != family_prequant_repo(hv480, "int8")
|
||||
# LTX has no measured quant recipe -> deliberately unwired.
|
||||
ltx = detect_video_family("unsloth/LTX-2.3-GGUF")
|
||||
assert family_prequant_repo(ltx, "int8") is None
|
||||
# LTX-2 base carries int8 + fp8 (both gate-validated); the 2.3 distilled weights get
|
||||
# their OWN repo via the variant table, keyed on the lowercased 2.3 base, because a
|
||||
# checkpoint baked from the base LTX-2 DiT fails base_model_id validation against 2.3.
|
||||
ltx = detect_video_family("Lightricks/LTX-2")
|
||||
assert family_prequant_repo(ltx, "int8") == "unsloth/LTX-2-FP8"
|
||||
assert family_prequant_repo(ltx, "fp8") == "unsloth/LTX-2-FP8"
|
||||
assert family_prequant_repo(ltx, "fp8", "Lightricks/LTX-2.3") == "unsloth/LTX-2.3-FP8"
|
||||
assert family_prequant_repo(ltx, "int8", "Lightricks/LTX-2.3") == "unsloth/LTX-2.3-FP8"
|
||||
# An unknown LTX variant falls back to the family default (the loader's base_model_id
|
||||
# check then refuses a mismatched checkpoint and dense-quantises).
|
||||
assert family_prequant_repo(ltx, "fp8", "someone/ltx-finetune") == "unsloth/LTX-2-FP8"
|
||||
|
||||
|
||||
def test_video_prequant_dual_expert_resolution():
|
||||
|
|
@ -375,3 +383,17 @@ def test_video_prequant_dual_expert_resolution():
|
|||
assert first.filename == "Wan2.2-T2V-A14B-INT8.pt"
|
||||
assert second.filename == "Wan2.2-T2V-A14B-INT8-2.pt"
|
||||
assert second.fallback_filename == "transformer_2_int8.pt"
|
||||
|
||||
|
||||
def test_video_prequant_ltx_variant_resolution():
|
||||
# The 2.3 base selects the 2.3 repo (its checkpoints stamp base_model_id
|
||||
# Lightricks/LTX-2.3); without a base the family default (base LTX-2 weights) resolves.
|
||||
from core.inference.diffusion_prequant import resolve_prequant_source
|
||||
|
||||
fam = detect_video_family("Lightricks/LTX-2")
|
||||
base = resolve_prequant_source(fam, "fp8")
|
||||
assert base.location == "unsloth/LTX-2-FP8"
|
||||
assert base.filename == "LTX-2-FP8.pt"
|
||||
v23 = resolve_prequant_source(fam, "int8", base_repo = "Lightricks/LTX-2.3")
|
||||
assert v23.location == "unsloth/LTX-2.3-FP8"
|
||||
assert v23.filename == "LTX-2.3-INT8.pt"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue