Expose the persistent conditioning cache in the start schema

DiffusionLoraConfig has carried cond_cache_dir for a while and the DiT trainer
acts on it, but DiffusionTrainingStartRequest omitted the field, so Pydantic
dropped it silently and every API-driven run fell back to the in-memory cache
that is rebuilt from scratch each time. The warm path skips loading the VAE and
the multi-GB text encoders on a rerun whose images, captions and resolution are
unchanged, so this was a real capability that could not be reached.

Contained like output_dir rather than left to the trainer subprocess's cwd,
since it is another directory the trainer writes to. Blank or omitted still
means the in-memory cache, so it must not resolve to the outputs root.
This commit is contained in:
Daniel Han 2026-07-27 05:48:07 +00:00
commit 4850fd239d
3 changed files with 38 additions and 0 deletions

View file

@ -798,6 +798,16 @@ class DiffusionTrainingStartRequest(BaseModel):
cache_variants: int = Field(
4, ge = 1, le = 16, description = "Frozen crop/flip variants per image in the latent cache"
)
cond_cache_dir: Optional[str] = Field(
None,
description = (
"Directory for the PERSISTENT conditioning cache (latents + text embeddings), reused "
"across runs: a rerun whose images, captions and resolution are unchanged skips "
"loading the VAE and the multi-GB text encoders entirely. Studio-relative names are "
"resolved under the Studio outputs root and absolute paths must stay inside it. "
"null or blank keeps the in-memory cache, which is rebuilt every run."
),
)
compile_transformer: Literal["off", "on", "auto"] = Field(
"auto", description = "Regional torch.compile of the transformer blocks"
)

View file

@ -1375,6 +1375,11 @@ async def start_diffusion_training(
from utils.paths import resolve_output_dir
config["data_dir"] = str(_resolve_diffusion_data_dir(config["data_dir"]))
config["output_dir"] = str(resolve_output_dir(config["output_dir"]))
# The persistent conditioning cache is another directory the TRAINER writes to, so it gets
# the same containment as output_dir rather than the trainer's cwd. Blank/None means the
# in-memory cache (the trainer's own "off"), so it must not resolve to the outputs root.
cond_cache = str(config.get("cond_cache_dir") or "").strip()
config["cond_cache_dir"] = str(resolve_output_dir(cond_cache)) if cond_cache else None
except ValueError as e:
raise HTTPException(status_code = 400, detail = str(e))

View file

@ -1911,3 +1911,26 @@ def test_gpu_load_admission_and_reserve_exclude_each_other():
svc.reserve()
svc.reserve()
svc.unreserve()
def test_route_start_carries_and_contains_the_conditioning_cache_dir(client):
# The trainer's persistent conditioning cache (cond_cache_dir) skips loading the VAE and the
# multi-GB text encoders on a rerun, but the start schema omitted the field, so Pydantic
# dropped it silently and every API-driven run fell back to the in-memory cache. It also has to
# be contained like output_dir: the trainer subprocess would otherwise resolve it against its
# own cwd.
from pathlib import Path
r = client.post("/api/train/diffusion/start", json = {**_BODY, "cond_cache_dir": "cond-cache"})
assert r.status_code == 200, r.text
resolved = client._fake.started_with["cond_cache_dir"]
assert Path(resolved).is_absolute()
assert Path(resolved).name == "cond-cache"
# Omitted or blank keeps the in-memory cache rather than resolving to the outputs root.
r = client.post("/api/train/diffusion/start", json = _BODY)
assert r.status_code == 200, r.text
assert client._fake.started_with["cond_cache_dir"] is None
r = client.post("/api/train/diffusion/start", json = {**_BODY, "cond_cache_dir": " "})
assert r.status_code == 200, r.text
assert client._fake.started_with["cond_cache_dir"] is None