Expose the persistent conditioning cache in the start schema
DiffusionLoraConfig has carried cond_cache_dir for a while and the DiT trainer acts on it, but DiffusionTrainingStartRequest omitted the field, so Pydantic dropped it silently and every API-driven run fell back to the in-memory cache that is rebuilt from scratch each time. The warm path skips loading the VAE and the multi-GB text encoders on a rerun whose images, captions and resolution are unchanged, so this was a real capability that could not be reached. Contained like output_dir rather than left to the trainer subprocess's cwd, since it is another directory the trainer writes to. Blank or omitted still means the in-memory cache, so it must not resolve to the outputs root.
This commit is contained in:
parent
7d15f202a4
commit
4850fd239d
3 changed files with 38 additions and 0 deletions
|
|
@ -798,6 +798,16 @@ class DiffusionTrainingStartRequest(BaseModel):
|
|||
cache_variants: int = Field(
|
||||
4, ge = 1, le = 16, description = "Frozen crop/flip variants per image in the latent cache"
|
||||
)
|
||||
cond_cache_dir: Optional[str] = Field(
|
||||
None,
|
||||
description = (
|
||||
"Directory for the PERSISTENT conditioning cache (latents + text embeddings), reused "
|
||||
"across runs: a rerun whose images, captions and resolution are unchanged skips "
|
||||
"loading the VAE and the multi-GB text encoders entirely. Studio-relative names are "
|
||||
"resolved under the Studio outputs root and absolute paths must stay inside it. "
|
||||
"null or blank keeps the in-memory cache, which is rebuilt every run."
|
||||
),
|
||||
)
|
||||
compile_transformer: Literal["off", "on", "auto"] = Field(
|
||||
"auto", description = "Regional torch.compile of the transformer blocks"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1375,6 +1375,11 @@ async def start_diffusion_training(
|
|||
from utils.paths import resolve_output_dir
|
||||
config["data_dir"] = str(_resolve_diffusion_data_dir(config["data_dir"]))
|
||||
config["output_dir"] = str(resolve_output_dir(config["output_dir"]))
|
||||
# The persistent conditioning cache is another directory the TRAINER writes to, so it gets
|
||||
# the same containment as output_dir rather than the trainer's cwd. Blank/None means the
|
||||
# in-memory cache (the trainer's own "off"), so it must not resolve to the outputs root.
|
||||
cond_cache = str(config.get("cond_cache_dir") or "").strip()
|
||||
config["cond_cache_dir"] = str(resolve_output_dir(cond_cache)) if cond_cache else None
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code = 400, detail = str(e))
|
||||
|
||||
|
|
|
|||
|
|
@ -1911,3 +1911,26 @@ def test_gpu_load_admission_and_reserve_exclude_each_other():
|
|||
svc.reserve()
|
||||
svc.reserve()
|
||||
svc.unreserve()
|
||||
|
||||
|
||||
def test_route_start_carries_and_contains_the_conditioning_cache_dir(client):
|
||||
# The trainer's persistent conditioning cache (cond_cache_dir) skips loading the VAE and the
|
||||
# multi-GB text encoders on a rerun, but the start schema omitted the field, so Pydantic
|
||||
# dropped it silently and every API-driven run fell back to the in-memory cache. It also has to
|
||||
# be contained like output_dir: the trainer subprocess would otherwise resolve it against its
|
||||
# own cwd.
|
||||
from pathlib import Path
|
||||
|
||||
r = client.post("/api/train/diffusion/start", json = {**_BODY, "cond_cache_dir": "cond-cache"})
|
||||
assert r.status_code == 200, r.text
|
||||
resolved = client._fake.started_with["cond_cache_dir"]
|
||||
assert Path(resolved).is_absolute()
|
||||
assert Path(resolved).name == "cond-cache"
|
||||
|
||||
# Omitted or blank keeps the in-memory cache rather than resolving to the outputs root.
|
||||
r = client.post("/api/train/diffusion/start", json = _BODY)
|
||||
assert r.status_code == 200, r.text
|
||||
assert client._fake.started_with["cond_cache_dir"] is None
|
||||
r = client.post("/api/train/diffusion/start", json = {**_BODY, "cond_cache_dir": " "})
|
||||
assert r.status_code == 200, r.text
|
||||
assert client._fake.started_with["cond_cache_dir"] is None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue