Merge branch 'video-wan' into video-hunyuan-gate

# Conflicts:
#	studio/backend/core/inference/video.py
#	studio/backend/tests/test_video_backend.py

studio/backend/tests/test_diffusion_train_perf.py

@@ -32,11 +32,16 @@ from core.training.diffusion_dit_trainer import (
 )
 from core.training.diffusion_train_common import (
     DiffusionLoraConfig,
+    LATENT_CACHE_OVER_BUDGET,
     _apply_perf_flags,
     _config_from_dict,
+    _latent_cache_forced,
+    _latent_cache_over_budget,
     _plan_cache_variants,
     _restore_perf_flags,
 )
+import core.training.diffusion_lora_trainer as sdxl_trainer
+import core.training.diffusion_train_common as train_common
 from core.training.diffusion_training_service import DiffusionTrainingService
 from models.training import DiffusionTrainingStartRequest, DiffusionTrainingStopRequest
 from routes.training import router as training_router
@@ -375,3 +380,90 @@ def test_perf_flags_tf32_off_clears_flags():
         torch.get_float32_matmul_precision(),
     )
     assert after == before
+
+
+# ── latent cache size gate ────────────────────────────────────────────────────
+class _FakeLatentDist:
+    def __init__(self, shape):
+        self.mean = torch.zeros(shape, dtype = torch.float32)
+        self.std = torch.ones(shape, dtype = torch.float32)
+
+
+class _FakeEncoded:
+    def __init__(self, shape):
+        self.latent_dist = _FakeLatentDist(shape)
+
+
+class _FakeVae:
+    # Minimal VAE stand-in: encode() returns a posterior of the requested latent shape so the
+    # builder measures a real per-variant byte size without a model load or image files.
+    def __init__(self, shape):
+        self._shape = shape
+
+    def encode(self, pixel_values):
+        return _FakeEncoded(self._shape)
+
+
+def _fake_planned_loader(path, resolution, center_crop, u_left, u_top, flip):
+    # The fake VAE ignores pixels; return a valid tensor + square SDXL time_ids.
+    tensor = torch.zeros(3, resolution, resolution, dtype = torch.float32)
+    return tensor, (resolution, resolution, 0, 0, resolution, resolution)
+
+
+def _build_fake_sdxl_cache(monkeypatch, num_images, latent_shape):
+    # center_crop + no flip collapses to one variant per image, so total_variants == num_images.
+    monkeypatch.setattr(sdxl_trainer, "_load_image_tensor_planned", _fake_planned_loader)
+    cfg = _cfg(cache_variants = 1, center_crop = True, random_flip = False).normalized()
+    return sdxl_trainer._build_sdxl_latent_cache(
+        _FakeVae(latent_shape),
+        1.0,
+        [f"img{i}.png" for i in range(num_images)],
+        cfg,
+        "cpu",
+        torch.float32,
+        None,
+        lambda: False,
+    )
+
+
+def test_latent_cache_over_budget_boundary():
+    # 32 bytes per variant x 4 variants = 128 bytes; exactly at budget is not "over".
+    assert _latent_cache_over_budget(32, 4, budget_bytes = 200) is False
+    assert _latent_cache_over_budget(32, 4, budget_bytes = 128) is False
+    assert _latent_cache_over_budget(32, 4, budget_bytes = 127) is True
+    # An empty plan can never overflow.
+    assert _latent_cache_over_budget(1_000_000, 0, budget_bytes = 1) is False
+
+
+def test_latent_cache_forced_env(monkeypatch):
+    monkeypatch.delenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", raising = False)
+    assert _latent_cache_forced() is False
+    monkeypatch.setenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", "1")
+    assert _latent_cache_forced() is True
+
+
+def test_sdxl_cache_built_under_budget(monkeypatch):
+    # Default (4 GiB) budget: a handful of tiny latents fits, so the full cache is returned.
+    monkeypatch.delenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", raising = False)
+    cache = _build_fake_sdxl_cache(monkeypatch, num_images = 3, latent_shape = (1, 4, 8, 8))
+    assert cache is not LATENT_CACHE_OVER_BUDGET and cache is not None
+    assert len(cache) == 3
+    assert all(len(variants) == 1 for variants in cache)
+
+
+def test_sdxl_cache_gated_over_budget(monkeypatch):
+    # A budget below one variant forces the gate on the first encode: the sentinel is returned
+    # so the caller keeps the VAE resident and encodes per step.
+    monkeypatch.delenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", raising = False)
+    monkeypatch.setattr(train_common, "_LATENT_CACHE_BUDGET_BYTES", 8)
+    cache = _build_fake_sdxl_cache(monkeypatch, num_images = 3, latent_shape = (1, 4, 8, 8))
+    assert cache is LATENT_CACHE_OVER_BUDGET
+
+
+def test_sdxl_cache_force_bypasses_gate(monkeypatch):
+    # An explicit force-on must be honoured verbatim even when the estimate is over budget.
+    monkeypatch.setenv("UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE", "1")
+    monkeypatch.setattr(train_common, "_LATENT_CACHE_BUDGET_BYTES", 8)
+    cache = _build_fake_sdxl_cache(monkeypatch, num_images = 3, latent_shape = (1, 4, 8, 8))
+    assert cache is not LATENT_CACHE_OVER_BUDGET and cache is not None
+    assert len(cache) == 3

studio/backend/tests/test_diffusion_training.py

@@ -441,6 +441,45 @@ def test_config_from_dict_epoch_mode_drops_max_steps_sentinel():
     assert cfg_explicit.train_steps == 25
 
 
+def test_permutation_sampler_covers_dataset_once_per_cycle():
+    # Every index must appear exactly once per cycle before any repeat (epoch-style pass),
+    # so a short run over a small dataset never leaves images unseen the way the old
+    # with-replacement draw did. Consecutive cycles must be reshuffled (differ).
+    import random
+
+    from core.training.diffusion_train_common import PermutationBatchSampler
+
+    n = 100
+    sampler = PermutationBatchSampler(n, random.Random(0))
+
+    # Draw exactly one cycle in batches of 3 (n not divisible by the batch, so a batch spans
+    # the cycle boundary); the first n indices must be a permutation of range(n).
+    drawn: list[int] = []
+    while len(drawn) < n:
+        drawn.extend(sampler.next_batch(3))
+    first_cycle = drawn[:n]
+    assert sorted(first_cycle) == list(range(n))  # each index once, none missing
+
+    # The next full cycle is also a permutation, and it is reshuffled (order differs).
+    fresh = PermutationBatchSampler(n, random.Random(0))
+    cycle_a = fresh.next_batch(n)
+    cycle_b = fresh.next_batch(n)
+    assert sorted(cycle_a) == list(range(n))
+    assert sorted(cycle_b) == list(range(n))
+    assert cycle_a != cycle_b  # cycles are reshuffled, not repeated in the same order
+
+    # A seed replays the exact index stream (determinism for reproducible runs).
+    replay = PermutationBatchSampler(n, random.Random(0))
+    assert replay.next_batch(n) == cycle_a
+
+    # A batch larger than the dataset refills across cycles so it never shrinks (batch shape
+    # preserved), even though it must then repeat indices within the batch.
+    big = PermutationBatchSampler(4, random.Random(1))
+    batch = big.next_batch(10)
+    assert len(batch) == 10
+    assert set(batch) == {0, 1, 2, 3}
+
+
 def test_route_start_accepts_zero_max_grad_norm(client):
     # 0 is the documented "disable clipping" value (the trainer skips clip_grad_norm_);
     # the request model must not reject it.
@@ -951,3 +990,16 @@ def test_runs_route_tolerates_bad_field_record(client, _isolated_runs_dir):
     assert r.status_code == 200, r.text
     adapters = [x["adapter"] for x in r.json()["runs"]]
     assert adapters == ["good"]  # the bad-field record was skipped, the good one remained
+
+
+def test_run_detail_route_non_object_record_is_404(client, _isolated_runs_dir):
+    # A valid-JSON but non-object record (a truncated / hand-edited [] file named with a real
+    # job id) makes DiffusionTrainingRunDetail(**rec) raise TypeError, not ValidationError; the
+    # detail route must shape-check like the list path and 404 instead of 500.
+    import json
+
+    job_id = "a" * 32
+    (_isolated_runs_dir / f"{job_id}.json").write_text(json.dumps([]))
+
+    r = client.get(f"/api/train/diffusion/runs/{job_id}")
+    assert r.status_code == 404, r.text

studio/backend/tests/test_video_backend.py

@@ -12,7 +12,12 @@ import types
 
 import pytest
 
-from core.inference.video import VideoBackend, get_video_backend, resolve_video_model_kind
+from core.inference.video import (
+    VideoBackend,
+    _detect_load_family,
+    get_video_backend,
+    resolve_video_model_kind,
+)
 from core.inference.video_families import VIDEO_CANCELLED_MSG, VIDEO_NOT_LOADED_MSG
 
 
@@ -454,6 +459,33 @@ def test_validate_gates_base_repo_and_local_paths(tmp_path):
         )
 
 
+def test_validate_rejects_gguf_repo_as_pipeline():
+    backend = VideoBackend()
+    # A -GGUF repo with no quant filename resolves to the pipeline kind and would
+    # only fail minutes later in from_pretrained, AFTER evicting the GPU owner.
+    with pytest.raises(ValueError, match = "pick one of its .gguf files"):
+        backend.validate_load_request("unsloth/LTX-2.3-GGUF")
+    with pytest.raises(ValueError, match = "pick one of its .gguf files"):
+        backend.validate_load_request("unsloth/Wan2.2-TI2V-5B-GGUF/")
+
+
+def test_detect_load_family_filename_fallback():
+    # Repo id alone carries the family.
+    fam = _detect_load_family("Lightricks/LTX-2", None, None)
+    assert fam is not None and fam.name == "ltx-2"
+    # Repo id is opaque but the picked filename carries it: fall back to the
+    # combined path so validate and _run_load agree on the family.
+    fam = _detect_load_family("someorg/quants", "ltx-2-19b-Q4_K_M.gguf", None)
+    assert fam is not None and fam.name == "ltx-2"
+    # No filename and no recognisable repo id: no family.
+    assert _detect_load_family("someorg/quants", None, None) is None
+    # An explicit override resolves by name/alias and skips the filename fallback:
+    # a bogus override stays None even when the filename would have matched.
+    fam = _detect_load_family("someorg/quants", "ltx-2-19b-Q4_K_M.gguf", "ltxv")
+    assert fam is not None and fam.name == "ltx-2"
+    assert _detect_load_family("someorg/quants", "ltx-2-19b-Q4_K_M.gguf", "bogus") is None
+
+
 def test_load_generate_unload_gguf(fake_runtime, tmp_path):
     backend = VideoBackend()
     status = _load_gguf(backend, tmp_path)

studio/backend/tests/test_video_routes.py

@@ -69,6 +69,7 @@ class _FakeBackend:
         model_path,
         *,
         gguf_filename = None,
+        base_repo = None,
         family_override = None,
         model_kind = None,
         transformer_quant = None,