diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py index 4e8c090a39..cab609b913 100644 --- a/studio/backend/core/inference/diffusion.py +++ b/studio/backend/core/inference/diffusion.py @@ -3,12 +3,17 @@ """Local diffusion (text-to-image) backend. -A torch-only singleton: it dequantises a single-file GGUF on-device via -``GGUFQuantizationConfig`` and pulls the rest of the pipeline (VAE, text -encoders, scheduler) from the matching base repo. torch/diffusers are imported -lazily so this stays importable in a no-torch runtime. ``begin_load`` runs on a -background thread; poll ``load_progress`` for the download bar. GPU-handoff -policy lives in the arbiter the routes call, not here. +A torch-only singleton that loads one of three "kinds" (see ``resolve_model_kind``): +a single-file GGUF transformer dequantised on-device via ``GGUFQuantizationConfig``, +a single-file safetensors transformer (e.g. fp8), or a full diffusers pipeline via +``from_pretrained`` (which re-applies an embedded quant config such as bnb-4bit). The +single-file kinds pull the rest of the pipeline (VAE, text encoders, scheduler) from +the matching base repo; the pipeline kind pulls everything from the repo itself. +Non-GGUF kinds are gated to the ``unsloth/*`` org (or a local path) for safety. + +torch/diffusers are imported lazily so this stays importable in a no-torch runtime. +``begin_load`` runs on a background thread; poll ``load_progress`` for the download +bar. GPU-handoff policy lives in the arbiter the routes call, not here. """ from __future__ import annotations @@ -39,14 +44,18 @@ from .diffusion_memory import ( apply_memory_plan, estimate_gguf_dense_mib, estimate_image_runtime_mib, + estimate_safetensors_dense_mib, file_size_mib, infer_gguf_quant_label, plan_diffusion_memory, snapshot_device_memory, ) from .diffusion_speed import ( + SPEED_DEFAULT, + SPEED_MAX, SPEED_OFF, apply_speed_optims, + compile_eligible, resolve_speed_mode, restore_backend_flags, snapshot_backend_flags, @@ -55,6 +64,16 @@ from .diffusion_attention import ( apply_attention_backend, select_attention_backend, ) +from . import diffusion_compile_cache as compile_cache +from . import diffusion_gguf_compile as gguf_compile +from .diffusion_eager_patches import ( + install_compile_safe_patches, + uninstall_patches, +) +from .diffusion_arch_patches import ( + install_arch_patches, + uninstall_arch_patches, +) from .diffusion_cache import apply_step_cache from .diffusion_precision import quantize_text_encoders from .diffusion_prequant import ( @@ -71,6 +90,110 @@ from .diffusion_transformer_quant import ( logger = get_logger(__name__) +# A load resolves to exactly one of these "kinds", which decide how the transformer +# (and the rest of the pipeline) is built: +# "gguf" -- a single-file GGUF transformer dequantised on-device via +# GGUFQuantizationConfig; the VAE / text encoders / scheduler come +# from the companion base diffusers repo. The original behaviour. +# "single_file" -- a single-file *.safetensors transformer loaded with from_single_file +# WITHOUT the GGUF dequant config (e.g. an fp8 checkpoint); companions +# still come from the base repo. +# "pipeline" -- a full diffusers repo loaded with pipeline_cls.from_pretrained(repo_id), +# which pulls every component (transformer included) and re-applies any +# embedded quantization_config (e.g. a bnb-4bit pipeline) automatically. +_MODEL_KINDS = frozenset({"gguf", "single_file", "pipeline"}) + + +def resolve_model_kind(gguf_filename: Optional[str], model_kind: Optional[str] = None) -> str: + """Classify a load request into one of ``_MODEL_KINDS``. + + An explicit ``model_kind`` wins (validated). Otherwise the kind is inferred from + the single-file name: a ``.gguf`` name is ``"gguf"``, any other single-file name is + ``"single_file"``, and the absence of a name is a full ``"pipeline"`` load. Pure and + network-free, so the route, validation, and load paths all agree on the kind.""" + if model_kind: + kind = model_kind.strip().lower() + if kind not in _MODEL_KINDS: + raise ValueError( + f"Unknown model_kind '{model_kind}'. Expected one of {sorted(_MODEL_KINDS)}." + ) + return kind + name = (gguf_filename or "").strip() + if not name: + return "pipeline" + if name.lower().endswith(".gguf"): + return "gguf" + return "single_file" + + +def _decode_b64_image(data: str, *, mode: str = "RGB") -> Any: + """Decode a base64 (optionally ``data:`` URL) image string to a PIL image. + + The image-conditioned workflows (img2img / inpaint / edit) transport the input + image and mask as base64 in the JSON request, so this is the single decode path. + A mask is decoded as single-channel ``L``; the source image as ``RGB``.""" + import base64 + import binascii + import io + + from PIL import Image + + raw = data.strip() + if raw.startswith("data:"): + # data:[][;base64], + _, _, raw = raw.partition(",") + try: + blob = base64.b64decode(raw, validate=False) + except (binascii.Error, ValueError) as exc: + raise ValueError(f"Invalid base64 image data: {exc}") from exc + try: + img = Image.open(io.BytesIO(blob)) + img.load() + except Exception as exc: # noqa: BLE001 — surfaced as a 400 to the client + raise ValueError(f"Could not decode image: {exc}") from exc + # Bound the decoded size. Every image-conditioned workflow (img2img / inpaint / upscale / + # reference / edit) decodes through here, so this single guard protects init, mask, and + # each reference image uniformly. PIL only WARNS in its 89-178MP "decompression bomb" soft + # zone and still loads (~0.5 GB RGB each, times up to 4 with multi-reference); cap the side + # well below that. 4096px covers txt2img's 2048 max, upscales, and normal outpaint canvases; + # anything larger is rejected with a clear 400 instead of risking an OOM. + max_side = 4096 + w, h = img.size + if w > max_side or h > max_side: + raise ValueError(f"Image is too large ({w}x{h}); maximum is {max_side}px per side.") + return img.convert(mode) + + +def _snap_to_multiple(img: Any, multiple: int = 16) -> Any: + """Resize a PIL image so both sides are multiples of ``multiple`` (rounded to nearest, + minimum one multiple), preserving content with a high-quality resample. + + Image-conditioned pipelines (Z-Image / Qwen / FLUX: 8x VAE downsample + 2x patch) reject + sizes that are not divisible by 16. Rather than error on an odd-sized upload, snap it so + the workflow just works; rounding to nearest keeps the rescale minimal/accurate.""" + from PIL import Image + + w, h = img.size + nw = max(multiple, int(round(w / multiple)) * multiple) + nh = max(multiple, int(round(h / multiple)) * multiple) + if (nw, nh) != (w, h): + img = img.resize((nw, nh), Image.LANCZOS) + return img + + +def _is_trusted_diffusion_repo(repo_id: str) -> bool: + """Whether a NON-GGUF load is allowed for ``repo_id``. + + Making ``gguf_filename`` optional opens a ``from_pretrained`` / ``from_single_file`` + on an arbitrary repo, which fetches and deserialises third-party weights. So the + non-GGUF paths are gated to the ``unsloth/*`` org (the curated safetensors models) and + to local paths the user explicitly pointed at (already on their disk). The GGUF path + is unchanged and stays open to any repo, as before.""" + if Path(repo_id).expanduser().exists(): + return True + return repo_id.strip().lower().startswith("unsloth/") + + @dataclass(frozen = True) class _LoadState: """Everything about the currently-loaded pipeline, swapped as one unit.""" @@ -87,6 +210,10 @@ class _LoadState: offload_policy: str = OFFLOAD_NONE vae_tiling: bool = False memory_mode: str = "auto" + # The resolved load kind: "gguf" | "single_file" | "pipeline". Surfaced in status so the + # UI can gate GGUF-only controls (the dense transformer_quant fast path only engages on + # the gguf kind; on single_file/pipeline it is a silent no-op). + kind: str = "gguf" # The opt-in speed profile (Phase 3). speed_mode: str = SPEED_OFF speed_optims: tuple = () @@ -104,6 +231,12 @@ class _LoadState: attention_backend: Optional[str] = None # Step cache engaged ("fbcache") or None. Opt-in, for many-step models. transformer_cache: Optional[str] = None + # Shared eager monkey-patches (diffusion_eager_patches) installed for this load (any + # non-off speed tier). Uninstalled on unload so a later `off` load is bit-identical. + eager_patched: bool = False + # Pre-warmed torch.compile cache context (diffusion_compile_cache.CacheContext) when a + # compiled tier ran, else None. Carries the per-key inductor dir + bundle for save/restore. + compile_cache_ctx: Any = None @dataclass @@ -180,6 +313,11 @@ class DiffusionBackend: # The callback mutates _gen and generate_progress() reads it, both lock-free, # so per-step progress polling stays live during a generation. self._gen: Optional[_GenState] = None + # Cache of image-conditioned workflow pipelines (img2img / inpaint) built via + # Pipeline.from_pipe around the loaded text-to-image pipe. They share its already + # resident modules (no extra VRAM, no reload), so we build each once per load and + # reuse it. Keyed by pipeline class name; cleared on unload with the base pipe. + self._aux_pipes: dict[str, Any] = {} @property def is_loaded(self) -> bool: @@ -249,31 +387,56 @@ class DiffusionBackend: *, gguf_filename: Optional[str] = None, family_override: Optional[str] = None, + model_kind: Optional[str] = None, ) -> DiffusionFamily: """Cheap, network-free validation shared by the route (before it evicts the - chat model) and both load paths, so an unloadable pick fails BEFORE the GPU - handoff. Raises ValueError for a missing gguf_filename or undetectable - family, and ValueError/FileNotFoundError for a bad local GGUF path. Touches - no GPU, network, or state.""" - if not gguf_filename: - raise ValueError( - "gguf_filename is required: this backend loads single-file GGUF checkpoints only." - ) + chat model) and the load paths, so an unloadable pick fails BEFORE the GPU + handoff. Resolves the load kind (gguf / single_file / pipeline), then raises + ValueError for a missing single-file name, a non-unsloth non-GGUF repo, or an + undetectable family, and ValueError/FileNotFoundError for a bad local path. + Touches no GPU, network, or state.""" + kind = resolve_model_kind(gguf_filename, model_kind) fam = detect_family(repo_id, family_override) if fam is None: raise ValueError( f"Could not infer a diffusion family for '{repo_id}'. Pass family_override (z-image)." ) + # Non-GGUF loads (a single-file safetensors transformer, or a full pipeline) + # are gated to the unsloth org or a local path -- they fetch + deserialise + # weights, so an arbitrary remote repo is rejected here, before any work. + if kind != "gguf" and not _is_trusted_diffusion_repo(repo_id): + raise ValueError( + f"Non-GGUF diffusion loads are restricted to unsloth/* repos (or a local " + f"path); got '{repo_id}'. Pass a gguf_filename to load a GGUF instead." + ) # Reject a bad LOCAL pick now (the same checks the load would hit later), so # the route never evicts a working chat model for a request that can't load. # A path-shaped repo_id (absolute / ~ / ./ / ..) is meant to be on disk, so a # missing one is an error here; a bare "org/name" id is a remote HF repo and # is left for the background load to resolve. local_root = Path(repo_id).expanduser() - if local_root.exists(): - resolve_local_gguf_child(local_root, gguf_filename) - elif repo_id.startswith(("/", "~", "./", "../")) or local_root.is_absolute(): - raise FileNotFoundError(f"Local model path does not exist: {repo_id}") + path_shaped = repo_id.startswith(("/", "~", "./", "../")) or local_root.is_absolute() + if kind in ("gguf", "single_file"): + if not gguf_filename: + raise ValueError( + f"a single-file checkpoint name is required for a '{kind}' load." + ) + if local_root.exists(): + resolve_local_gguf_child(local_root, gguf_filename) + elif path_shaped: + raise FileNotFoundError(f"Local model path does not exist: {repo_id}") + else: # pipeline + if gguf_filename: + raise ValueError( + "a 'pipeline' load takes a full diffusers repo, not a single-file name." + ) + if local_root.exists(): + if not (local_root / "model_index.json").exists(): + raise FileNotFoundError( + f"Local pipeline directory has no model_index.json: {repo_id}" + ) + elif path_shaped: + raise FileNotFoundError(f"Local model path does not exist: {repo_id}") return fam # ── Background load + progress ───────────────────────────────────────── @@ -296,10 +459,14 @@ class DiffusionBackend: attention_backend: Optional[str] = None, transformer_cache: Optional[str] = None, transformer_cache_threshold: Optional[float] = None, + model_kind: Optional[str] = None, ) -> dict[str, Any]: """Validate, then run the (slow) load on a daemon thread. Returns at once.""" fam = self.validate_load_request( - repo_id, gguf_filename = gguf_filename, family_override = family_override + repo_id, + gguf_filename = gguf_filename, + family_override = family_override, + model_kind = model_kind, ) with self._lock: @@ -333,6 +500,7 @@ class DiffusionBackend: attention_backend = attention_backend, transformer_cache = transformer_cache, transformer_cache_threshold = transformer_cache_threshold, + model_kind = model_kind, _load_token = token, ), daemon = True, @@ -346,12 +514,18 @@ class DiffusionBackend: # calls) so begin_load returns instantly; the bar shows raw bytes until # the total lands. This is the only writer of _loading's fields here. fam = detect_family(kwargs["repo_id"], kwargs.get("family_override")) - base = _resolve_base_repo( - kwargs["repo_id"], kwargs.get("base_repo"), fam, kwargs.get("hf_token") - ) + kind = resolve_model_kind(kwargs.get("gguf_filename"), kwargs.get("model_kind")) + if kind == "pipeline": + # The full pipeline IS the repo: from_pretrained pulls every component + # (transformer included) from it, so the base repo is the repo itself. + base = kwargs["repo_id"] + else: + base = _resolve_base_repo( + kwargs["repo_id"], kwargs.get("base_repo"), fam, kwargs.get("hf_token") + ) kwargs["base_repo"] = base expected, base_files = self._estimate_download_bytes( - kwargs["repo_id"], kwargs.get("gguf_filename"), base, kwargs.get("hf_token") + kwargs["repo_id"], kwargs.get("gguf_filename"), base, kwargs.get("hf_token"), kind = kind ) loading = self._loading if loading is not None: @@ -390,7 +564,11 @@ class DiffusionBackend: if loading is None: return _progress("ready" if self._state is not None else None) - downloaded = self._cache_bytes(loading.repo_id) + self._cache_bytes(loading.base_repo) + # Sum the checkpoint repo + companion base cache. For a full-pipeline load the + # base IS the repo, so count it once (else the bar double-counts to "finalizing"). + downloaded = self._cache_bytes(loading.repo_id) + if loading.base_repo and loading.base_repo != loading.repo_id: + downloaded += self._cache_bytes(loading.base_repo) expected = loading.expected_bytes # Downloads done but pipeline still dequantising / moving to GPU. The cache # scan can slightly exceed the estimate (extra cached quants, blob padding), @@ -402,16 +580,33 @@ class DiffusionBackend: @staticmethod def _estimate_download_bytes( - repo_id: str, gguf_filename: Optional[str], base_repo: str, hf_token: Optional[str] + repo_id: str, + gguf_filename: Optional[str], + base_repo: str, + hf_token: Optional[str], + *, + kind: str = "gguf", ) -> tuple[int, list[str]]: """Total download size for the progress bar, plus the base-repo files to - fetch (the prefetch reuses this list, so the base is listed only once).""" + fetch (the prefetch reuses this list, so the base is listed only once). + + For a ``pipeline`` load the whole repo IS the pipeline (``base_repo`` is the + repo itself), so the transformer/ subfolder is INCLUDED -- unlike the GGUF / + single-file paths, where the transformer is the single file and the base repo + supplies only the companions.""" from huggingface_hub import HfApi api = HfApi() total = 0 base_files: list[str] = [] try: + if kind == "pipeline": + info = api.model_info(repo_id, files_metadata = True, token = hf_token) + for s in info.siblings: + if _pipeline_file_downloaded(s.rfilename): + base_files.append(s.rfilename) + total += s.size or 0 + return total, base_files if gguf_filename: info = api.model_info(repo_id, files_metadata = True, token = hf_token) total += sum(s.size or 0 for s in info.siblings if s.rfilename == gguf_filename) @@ -460,14 +655,21 @@ class DiffusionBackend: attention_backend: Optional[str] = None, transformer_cache: Optional[str] = None, transformer_cache_threshold: Optional[float] = None, + model_kind: Optional[str] = None, _load_token: Optional[int] = None, ) -> dict[str, Any]: # Validate first (cheap, no torch/diffusers) so a direct call with a bad # family fails with ValueError even in a no-diffusers runtime. fam = self.validate_load_request( - repo_id, gguf_filename = gguf_filename, family_override = family_override + repo_id, + gguf_filename = gguf_filename, + family_override = family_override, + model_kind = model_kind, ) - base = _resolve_base_repo(repo_id, base_repo, fam, hf_token) + kind = resolve_model_kind(gguf_filename, model_kind) + # For a full pipeline the repo itself supplies every component, so it is its + # own base; the single-file kinds resolve the companion base diffusers repo. + base = repo_id if kind == "pipeline" else _resolve_base_repo(repo_id, base_repo, fam, hf_token) target = self._resolve_device_target(fam) device, dtype = target.device, target.dtype @@ -492,7 +694,13 @@ class DiffusionBackend: # checkpoints never sit in VRAM at once. self._unload_locked() - gguf_path = self._resolve_gguf_path(repo_id, gguf_filename, hf_token) + # The single-file kinds resolve a checkpoint path (GGUF or safetensors); + # the pipeline kind has none (from_pretrained pulls the repo directly). + single_file_path = ( + self._resolve_gguf_path(repo_id, gguf_filename, hf_token) + if kind in ("gguf", "single_file") + else None + ) transformer_cls = getattr(diffusers, fam.transformer_class) pipeline_cls = getattr(diffusers, fam.pipeline_class) @@ -501,18 +709,30 @@ class DiffusionBackend: # dense bf16 transformer must fit resident, so the fast path is offered only # when the plan is `none`. plan = self._plan_memory( - target, gguf_path, gguf_filename, base, fam, memory_mode, cpu_offload + target, + single_file_path, + gguf_filename, + base, + fam, + memory_mode, + cpu_offload, + kind = kind, + repo_id = repo_id, ) # Opt-in fast path: load the DENSE bf16 transformer and torchao-quantise it # (int8 / fp8 / fp4 tensor cores), which beats GGUF's bf16-rate per-matmul # dequant on both speed and quality, at the cost of a higher-memory dense # load. Gated on CUDA + bf16 + a resident fit; ANY failure (unsupported arch - # / scheme, OOM, partial quant) falls back to the GGUF build below. + # / scheme, OOM, partial quant) falls back to the GGUF build below. Only the + # GGUF kind offers it: it materialises the dense bf16 transformer from the + # base repo, which the safetensors kinds (a single-file or already-quantized + # pipeline) do not have. pipe = None transformer_quant_engaged = None if ( - normalize_transformer_quant(transformer_quant) is not None + kind == "gguf" + and normalize_transformer_quant(transformer_quant) is not None and dense_transformer_supported(target) and plan.offload_policy == OFFLOAD_NONE ): @@ -539,30 +759,45 @@ class DiffusionBackend: clear_gpu_cache() if pipe is None: - # Default: dequantise the single-file GGUF transformer on-device; the - # VAE / text-encoder / scheduler come from the base diffusers repo - # (GGUF is transformer-only). - transformer = transformer_cls.from_single_file( - gguf_path, - quantization_config = diffusers.GGUFQuantizationConfig(compute_dtype = dtype), - torch_dtype = dtype, - config = base, - subfolder = "transformer", - # Forward the token: the config is fetched from the (possibly gated) - # base repo before from_pretrained gets a chance to authenticate. - token = hf_token, - ) + if kind == "pipeline": + # Full diffusers repo: from_pretrained pulls every component + # (transformer + VAE + text encoders + scheduler) from the repo + # and re-applies any embedded quantization_config (e.g. bnb-4bit), + # so a pre-quantized pipeline reloads quantized with no extra config. + pipe_kwargs: dict[str, Any] = {"torch_dtype": dtype} + if hf_token: + pipe_kwargs["token"] = hf_token + pipe = pipeline_cls.from_pretrained(repo_id, **pipe_kwargs) + else: + # Single-file transformer; the VAE / text-encoder / scheduler come + # from the base diffusers repo (the single file is transformer-only). + sf_kwargs: dict[str, Any] = { + "torch_dtype": dtype, + "config": base, + "subfolder": "transformer", + # Forward the token: the config is fetched from the (possibly + # gated) base repo before from_pretrained can authenticate. + "token": hf_token, + } + if kind == "gguf": + # Dequantise the GGUF transformer on-device at the compute dtype. + sf_kwargs["quantization_config"] = diffusers.GGUFQuantizationConfig( + compute_dtype = dtype + ) + # A safetensors single-file (e.g. fp8) carries its own dtype, so no + # GGUF dequant config is passed. + transformer = transformer_cls.from_single_file(single_file_path, **sf_kwargs) - pipe_kwargs: dict[str, Any] = {"torch_dtype": dtype, "transformer": transformer} - if hf_token: - pipe_kwargs["token"] = hf_token - pipe = pipeline_cls.from_pretrained(base, **pipe_kwargs) + pipe_kwargs = {"torch_dtype": dtype, "transformer": transformer} + if hf_token: + pipe_kwargs["token"] = hf_token + pipe = pipeline_cls.from_pretrained(base, **pipe_kwargs) # Resolve the effective speed mode: GGUF models default to the # near-lossless `default` profile (compile is ~2.2x and sits below # the quant noise floor), dense models stay bit-identical `off`. An # explicit speed_mode (incl. "off") is honored verbatim. - effective_speed = resolve_speed_mode(speed_mode, is_gguf = bool(gguf_filename)) + effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf") # Opt-in speed optims run BEFORE placement (channels_last / compile # must precede CPU offload). Snapshot the process-wide backend flags # first so unload can restore them: TF32 / cudnn.benchmark are global, @@ -591,53 +826,129 @@ class DiffusionBackend: quant_active = transformer_quant_engaged is not None, logger = logger, ) - speed_applied = apply_speed_optims( - pipe, - target, - is_gguf = bool(gguf_filename), - family = fam, - speed_mode = effective_speed, - cache_active = cache_engaged is not None, - logger = logger, - ) - # Quantise the dense companion text encoder(s) (opt-in fp8 / nvfp4), - # also before placement so the offload hooks move the smaller weights. - te_quant = quantize_text_encoders( - pipe, - target, - mode = text_encoder_quant, - logger = logger, - ) + # Install the shared compile-safe eager patches (fused RMSNorm / + # AdaLayerNorm) for any active speed tier. They are class-level, idempotent + # and math-equivalent (FMA / fused -> neutral under compile, equal-or-more + # accurate), so they help eager AND compiled runs. The bit-identical `off` + # reference path must run with them UNINSTALLED, so uninstall there. + # + # Everything from here to the _LoadState commit mutates PROCESS-WIDE state + # (class patches, TORCHINDUCTOR_CACHE_DIR, backend flags). _unload_locked only + # reverses it via _state, so a failure BEFORE the commit would leak it (and + # break the next `off` load's bit-identity). Guard the whole block: on any + # pre-commit failure, restore everything; on success the commit transfers + # ownership to _state and _unload_locked takes over. + # The GGUF-specific speed lever (compiled dequant) applies only when the + # GGUF transformer was ACTUALLY loaded. On the dense torchao-quant + # fast path (fp8 / int8 / fp4) `gguf_filename` is still set as the fallback, + # but `pipe.transformer` is dense (no GGUFLinear), and those schemes need the + # REGIONAL block compile (dynamic quant is ~30x slower eager), not the GGUF + # dequant compile -- so treat the transformer as non-GGUF here. The + # safetensors kinds (single_file / pipeline) likewise have no GGUFLinear. + gguf_transformer = kind == "gguf" and transformer_quant_engaged is None - # Apply the placement planned above (from MEASURED free device memory vs - # the model's estimated resident size). apply_memory_plan returns the - # (policy, tiling) ACTUALLY engaged (it may fall back to whole-module - # offload, and tiling is a no-op on a pipeline with no tiling control), so - # status stays honest. The dense fast path already placed the pipe resident; - # for the `none` policy this is an idempotent re-placement. - effective_policy, effective_tiling = apply_memory_plan( - pipe, plan, device = device, logger = logger - ) + eager_patched = False + compile_ctx = None + state_committed = False + try: + if effective_speed != SPEED_OFF: + install_compile_safe_patches() + # Per-arch compile-safe fusions (qwen _modulate / z-image residual + # addcmul, etc.). Also neutral under compile, so on for every active + # tier; tracked by the same eager_patched flag for uninstall. + install_arch_patches() + eager_patched = True + else: + uninstall_patches() + uninstall_arch_patches() - self._state = _LoadState( - pipe = pipe, - family = fam, - repo_id = repo_id, - base_repo = base, - device = device, - dtype = str(dtype).replace("torch.", ""), - cpu_offload = effective_policy != OFFLOAD_NONE, - offload_policy = effective_policy, - vae_tiling = effective_tiling, - memory_mode = plan.requested_mode, - speed_mode = effective_speed, - speed_optims = tuple(k for k, v in speed_applied.items() if v), - backend_flags_before = backend_flags_before, - text_encoder_quant = te_quant, - transformer_quant = transformer_quant_engaged, - attention_backend = attention_engaged, - transformer_cache = cache_engaged, - ) + # Pre-warmed torch.compile cache (Mega-cache): when a compiled tier will + # run, point inductor at a per-fingerprint dir and load a matching bundle + # BEFORE the first compiled forward, so the one-time 25-58s compile can be + # paid once (by us / a first run) and reused. A miss is silent -> local + # compile, exactly as today. + if effective_speed in (SPEED_DEFAULT, SPEED_MAX) and compile_eligible( + target, is_gguf = gguf_transformer, family = fam + ): + compile_ctx = compile_cache.begin( + family = fam.name, + transformer = getattr(pipe, "transformer", None), + dtype = getattr(target, "dtype", None), + quant = transformer_quant_engaged, + attention_backend = attention_engaged, + compile_kwargs = { + "fullgraph": cache_engaged is None, + "dynamic": effective_speed != SPEED_MAX, + "mode": "max-autotune-no-cudagraphs" + if effective_speed == SPEED_MAX + else "default", + }, + logger = logger, + ) + + speed_applied = apply_speed_optims( + pipe, + target, + is_gguf = gguf_transformer, + family = fam, + speed_mode = effective_speed, + cache_active = cache_engaged is not None, + logger = logger, + ) + # Quantise the dense companion text encoder(s) (opt-in fp8 / nvfp4), + # also before placement so the offload hooks move the smaller weights. + te_quant = quantize_text_encoders( + pipe, + target, + mode = text_encoder_quant, + logger = logger, + ) + + # Apply the placement planned above (from MEASURED free device memory vs + # the model's estimated resident size). apply_memory_plan returns the + # (policy, tiling) ACTUALLY engaged (it may fall back to whole-module + # offload, and tiling is a no-op on a pipeline with no tiling control), so + # status stays honest. The dense fast path already placed the pipe + # resident; for the `none` policy this is an idempotent re-placement. + effective_policy, effective_tiling = apply_memory_plan( + pipe, plan, device = device, logger = logger + ) + + self._state = _LoadState( + pipe = pipe, + family = fam, + repo_id = repo_id, + base_repo = base, + device = device, + dtype = str(dtype).replace("torch.", ""), + kind = kind, + cpu_offload = effective_policy != OFFLOAD_NONE, + offload_policy = effective_policy, + vae_tiling = effective_tiling, + memory_mode = plan.requested_mode, + speed_mode = effective_speed, + speed_optims = tuple(k for k, v in speed_applied.items() if v), + backend_flags_before = backend_flags_before, + text_encoder_quant = te_quant, + transformer_quant = transformer_quant_engaged, + attention_backend = attention_engaged, + transformer_cache = cache_engaged, + eager_patched = eager_patched, + compile_cache_ctx = compile_ctx, + ) + state_committed = True + finally: + # Pre-commit failure: nothing owns the process-wide mutations yet, so + # roll them back here (symmetric with _unload_locked). + if not state_committed: + restore_backend_flags(backend_flags_before) + compile_cache.restore(compile_ctx) + # apply_speed_optims may have installed the compiled GGUF dequant + # before a later step failed; uninstall is idempotent. + gguf_compile.uninstall_all() + if eager_patched: + uninstall_patches() + uninstall_arch_patches() logger.info( "diffusion.loaded: repo=%s base=%s device=%s offload=%s tiling=%s reasons=%s", @@ -732,28 +1043,47 @@ class DiffusionBackend: def _plan_memory( self, target: DiffusionDeviceTarget, - gguf_path: str, + single_file_path: Optional[str], gguf_filename: Optional[str], base: str, fam: DiffusionFamily, memory_mode: Optional[str], cpu_offload: bool, + *, + kind: str = "gguf", + repo_id: Optional[str] = None, ): """Build the memory plan for this load: snapshot free device memory and estimate the model's resident footprint, then let the planner pick an offload policy + VAE memory savers. Kept on the backend so the cached base - repo (companion text-encoder / VAE) feeds the size estimate.""" + repo (companion text-encoder / VAE) feeds the size estimate. + + The size estimate is per-kind: a GGUF dequantises (a 4-bit file ~4x), a + safetensors single-file loads near its on-disk size, and a full pipeline is + one cached download (transformer + companions) that is already compressed.""" device_memory = snapshot_device_memory(target) - transformer_dense = estimate_gguf_dense_mib( - file_size_mib(gguf_path), infer_gguf_quant_label(gguf_filename) - ) - # The companion components (VAE + text encoders) load near their on-disk - # size; sum whatever the prefetch already placed in the base-repo cache. - companion = self._cache_bytes(base) - companion_mib = int(companion // (1024 * 1024)) if companion else None - model_dense_mib = None - if transformer_dense is not None: - model_dense_mib = transformer_dense + (companion_mib or 0) + if kind == "pipeline": + # The whole repo (transformer + companions) is one cached download; the + # cached bytes are the resident estimate (bnb-4bit / fp8 stay compressed). + cached = self._cache_bytes(repo_id) if repo_id else 0 + cached_mib = int(cached // (1024 * 1024)) if cached else None + model_dense_mib = estimate_safetensors_dense_mib(cached_mib) + companion_mib = None + else: + if kind == "single_file": + # Safetensors single-file: no dequant expansion (it carries its dtype). + transformer_dense = estimate_safetensors_dense_mib(file_size_mib(single_file_path)) + else: + transformer_dense = estimate_gguf_dense_mib( + file_size_mib(single_file_path), infer_gguf_quant_label(gguf_filename) + ) + # The companion components (VAE + text encoders) load near their on-disk + # size; sum whatever the prefetch already placed in the base-repo cache. + companion = self._cache_bytes(base) + companion_mib = int(companion // (1024 * 1024)) if companion else None + model_dense_mib = None + if transformer_dense is not None: + model_dense_mib = transformer_dense + (companion_mib or 0) runtime_headroom = estimate_image_runtime_mib(width = None, height = None, family = fam.name) return plan_diffusion_memory( target = target, @@ -765,6 +1095,53 @@ class DiffusionBackend: explicit_offload = cpu_offload, ) + def _workflow_pipe(self, state: _LoadState, class_name: Optional[str], workflow: str) -> Any: + """The diffusers pipeline for an image-conditioned ``workflow``, built once and + cached. ``Pipeline.from_pipe`` re-wires the loaded text-to-image pipe's resident + modules (transformer/VAE/text-encoder, incl. any compiled/quantised state) into + the workflow pipeline class, so there is no extra VRAM and no reload. Raises a + clear ValueError when the family does not support the workflow.""" + if not class_name: + raise ValueError( + f"{workflow} is not supported for the '{state.family.name}' model family." + ) + cached = self._aux_pipes.get(class_name) + if cached is not None: + return cached + import diffusers + + # torch_dtype=None is load-bearing: diffusers' from_pipe defaults torch_dtype to + # torch.float32 and then runs new_pipeline.to(dtype=float32) over EVERY component. + # That recast (a) needlessly upcasts the reused bf16 modules and (b) hard-crashes + # on the dense-quant fast path -- a torchao-quantized + torch.compiled transformer + # has tensor-subclass Linear weights that torch.nn.Module._apply cannot swap_tensors + # ("Couldn't swap Linear.weight"). Passing None makes from_pipe skip the cast and + # reuse the resident modules AT THEIR LOADED dtype, which is the whole point of + # from_pipe (component reuse, no reload, no extra VRAM). + pipe = getattr(diffusers, class_name).from_pipe(state.pipe, torch_dtype = None) + self._aux_pipes[class_name] = pipe + return pipe + + @staticmethod + def _align_vae_dtype(pipe: Any) -> None: + """Cast the VAE to the transformer's compute dtype before an image-conditioned + call. The img2img/inpaint pipelines VAE-encode the input image at the text- + encoder dtype (bf16), but a prior txt2img DECODE may have left the shared VAE + upcast to fp32 (its ``force_upcast`` path), so the encode would mismatch + (bf16 image vs fp32 VAE). Re-aligning here is safe: our families run bf16 or + fp32 only (the fp16 guard promotes fp16), and a later txt2img decode re-upcasts + as needed. Best-effort; a no-op when already aligned.""" + transformer = getattr(pipe, "transformer", None) + vae = getattr(pipe, "vae", None) + if transformer is None or vae is None: + return + try: + target_dtype = transformer.dtype + if next(vae.parameters()).dtype != target_dtype: + vae.to(dtype=target_dtype) + except (StopIteration, AttributeError, RuntimeError): + pass + def generate( self, *, @@ -779,8 +1156,22 @@ class DiffusionBackend: guidance: float = 0.0, seed: Optional[int] = None, batch_size: int = 1, + # Image-conditioned workflows (base64 / data-URL): an init image alone selects + # img2img; an init image + mask selects inpaint. ``strength`` is the img2img/ + # inpaint denoise strength (0 = keep source, 1 = full redraw). None = txt2img. + init_image: Optional[str] = None, + mask_image: Optional[str] = None, + strength: Optional[float] = None, + # Upscale (hires fix): a factor > 1 with an init image enlarges the input and + # re-denoises it at low strength to paint detail at the higher resolution. + upscale: Optional[float] = None, + # Reference workflow (FLUX.2): ADDITIONAL reference images beyond ``init_image``. The + # pipeline accepts a list, so multiple references can be combined (subject + style, + # character + scene). Ignored by non-reference workflows. + reference_images: Optional[list[str]] = None, ) -> dict[str, Any]: import torch + from PIL import Image # A per-generation cancel Event: unload()/a superseding load set THIS event # (registered under _lock below) to abort just this denoise. _generate_lock @@ -810,10 +1201,98 @@ class DiffusionBackend: seed = int(seed) generator.manual_seed(seed) + # Select the pipeline for this workflow. txt2img uses the loaded pipe; + # img2img/inpaint reuse its resident modules via from_pipe (no reload); + # an edit model's OWN loaded pipe is already the edit pipeline. + pipe = state.pipe + init_pil = mask_pil = None + ref_extra: list = [] + if getattr(state.family, "edit", False): + # Instruction editing: the loaded pipe is the edit pipeline. It always + # needs an input image; the prompt is the edit instruction. No mask, no + # from_pipe (the model has no plain text-to-image mode). + if init_image is None: + raise ValueError( + f"{state.family.name} is an image-editing model: provide an input image." + ) + workflow = "edit" + init_pil = _decode_b64_image(init_image, mode = "RGB") + elif mask_image is not None and init_image is not None: + workflow = "inpaint" + pipe = self._workflow_pipe(state, state.family.inpaint_pipeline_class, workflow) + init_pil = _decode_b64_image(init_image, mode = "RGB") + mask_pil = _decode_b64_image(mask_image, mode = "L") + elif init_image is not None and upscale is not None and upscale > 1.0: + # Upscale (hires fix): enlarge the input with Lanczos, then re-run the + # img2img pipeline on it at a low denoise strength so the transformer + # adds high-frequency detail without redrawing the content. Shares the + # img2img pipeline/modules via from_pipe (no extra VRAM, no reload). + workflow = "upscale" + pipe = self._workflow_pipe(state, state.family.img2img_pipeline_class, workflow) + init_pil = _decode_b64_image(init_image, mode = "RGB") + iw, ih = init_pil.size + # Cap the factor, THEN cap the absolute output: a large input times the + # factor (e.g. 1024 at 4x = 4096, or a big upload) would otherwise OOM the + # VAE/transformer. Bound the longest side to 2048 (txt2img's own max), + # scaling both dims to keep the aspect ratio; round to a multiple of 16 + # (VAE downsample + patch size require it for our families). + factor = max(1.0, min(float(upscale), 4.0)) + tw_f, th_f = iw * factor, ih * factor + max_side = 2048 + fit = min(1.0, max_side / max(tw_f, th_f)) + tw = max(16, int(round(tw_f * fit / 16.0)) * 16) + th = max(16, int(round(th_f * fit / 16.0)) * 16) + init_pil = init_pil.resize((tw, th), Image.LANCZOS) + if strength is None: + # Hires-fix default: low enough to preserve content, high enough to + # synthesise new detail at the higher resolution. + strength = 0.35 + elif getattr(state.family, "reference", False) and init_image is not None: + # FLUX.2-style reference conditioning: the loaded pipe (Flux2KleinPipeline) + # takes the reference image directly via its `image` arg and generates a + # fresh image at the REQUESTED size, guided by both the prompt and the + # reference. No from_pipe (the loaded pipe already supports it), no strength + # (reference-conditioning, not a denoise blend), and the output size comes + # from the sliders (the pipeline resizes the reference to ~1MP itself). + # Checked AFTER inpaint/upscale so a mask/upscale request on a reference + # family (FLUX.2-klein also has an inpaint pipeline) still routes correctly. + workflow = "reference" + init_pil = _decode_b64_image(init_image, mode = "RGB") + # Additional references (FLUX.2 accepts a list): decode them so the + # conditioning combines all of them. Capped to keep VRAM bounded. + ref_extra = [ + _decode_b64_image(x, mode = "RGB") + for x in (reference_images or [])[:3] + ] + elif init_image is not None: + workflow = "img2img" + pipe = self._workflow_pipe(state, state.family.img2img_pipeline_class, workflow) + init_pil = _decode_b64_image(init_image, mode = "RGB") + else: + workflow = "txt2img" + # Auto-resize odd-sized inputs to a multiple of 16 for the workflows whose + # OUTPUT size is taken from the input image (img2img / inpaint / extend / edit), + # so an upload like 186px tall no longer fails the pipeline's divisibility check. + # txt2img/reference use the validated slider size; upscale already produced a /16 + # target. The mask is matched to the snapped image so inpaint stays aligned. + if init_pil is not None and workflow in ("img2img", "inpaint", "edit"): + init_pil = _snap_to_multiple(init_pil, 16) + if mask_pil is not None and mask_pil.size != init_pil.size: + from PIL import Image as _PILImage + + mask_pil = mask_pil.resize(init_pil.size, _PILImage.NEAREST) + if init_pil is not None: + # Keep the VAE encode dtype consistent with the input image. + self._align_vae_dtype(pipe) + + # Pipelines vary in which kwargs they accept (img2img derives size from the + # input image and may reject width/height; a distilled pipe may take no + # negative prompt or step callback), so gate every optional kwarg on the + # actual signature. + call_params = inspect.signature(pipe.__call__).parameters + kwargs: dict[str, Any] = { "prompt": prompt, - "width": width, - "height": height, "num_inference_steps": steps, # Most pipelines take guidance via "guidance_scale"; Qwen-Image # uses "true_cfg_scale" (its distilled guidance is off). @@ -823,10 +1302,33 @@ class DiffusionBackend: # share this call's seed, drawn sequentially from one generator. "num_images_per_prompt": batch_size, } - # Pipelines vary in which kwargs they accept (a distilled pipeline may - # take neither a negative prompt nor a step callback), so only pass - # those where the signature has them. - call_params = inspect.signature(state.pipe.__call__).parameters + if init_pil is not None: + # Reference with extra images passes the whole list (FLUX.2 combines them); + # every other workflow takes the single image. + kwargs["image"] = [init_pil, *ref_extra] if ref_extra else init_pil + if mask_pil is not None and "mask_image" in call_params: + kwargs["mask_image"] = mask_pil + if strength is not None and "strength" in call_params: + kwargs["strength"] = strength + # width/height. txt2img uses the requested slider size. Image-conditioned + # pipes must use the INPUT IMAGE's own size, NOT the slider: the output is + # the redrawn/extended input, and the denoise builds latents from the image, + # so a slider size that differs from the image mismatches (e.g. a 1536px + # outpaint vs a 1024 slider -> "tensor a (128) must match tensor b (192)"). + # Many img2img/inpaint pipelines drop width/height entirely; pass them only + # when accepted, derived from the image so they are always consistent. + if workflow in ("txt2img", "reference"): + # txt2img and FLUX.2 reference both generate at the REQUESTED size; the + # reference pipe resizes the conditioning image itself, so it must not be + # pinned to the input image's size like img2img/inpaint/upscale are. + kwargs["width"] = width + kwargs["height"] = height + elif init_pil is not None: + iw, ih = init_pil.size + if "width" in call_params: + kwargs["width"] = iw + if "height" in call_params: + kwargs["height"] = ih if negative_prompt and "negative_prompt" in call_params: kwargs["negative_prompt"] = negative_prompt @@ -854,13 +1356,20 @@ class DiffusionBackend: # inference_mode is strictly faster than the no_grad diffusers # uses internally and numerically identical for inference. with torch.inference_mode(): - images = state.pipe(**kwargs).images + images = pipe(**kwargs).images finally: self._gen = None # A cancelled denoise returns early with a partial/garbage image; # don't hand it back to be persisted. if cancel.is_set(): raise RuntimeError("Diffusion generation was cancelled.") + # The first compiled generation just paid the compile cost; persist the + # warm torch.compile cache bundle when saving is enabled (distributor / + # first-run warm). Idempotent + best-effort -- never fails a generation. + try: + compile_cache.save(state.compile_cache_ctx, logger = logger) + except Exception: # noqa: BLE001 — cache persistence is best-effort + pass # Return the PIL images (not yet encoded): the route embeds each # image's recipe and persists it via the gallery. return {"images": list(images), "seed": int(seed), "repo_id": state.repo_id} @@ -916,6 +1425,20 @@ class DiffusionBackend: # Restore the process-wide backend flags (TF32 / cudnn.benchmark) this load # may have flipped, so the next `off` load is bit-identical again. restore_backend_flags(state.backend_flags_before) + # Restore TORCHINDUCTOR_CACHE_DIR and uninstall the shared eager patches, so a + # later `off` load runs the bit-identical reference path. Both are idempotent. + compile_cache.restore(state.compile_cache_ctx) + # Uninstall the GGUF dequant accelerators (compiled dequant / global weight + # buffer) this load may have installed, so a later `off` load runs the stock, + # bit-identical dequant. Idempotent. + gguf_compile.uninstall_all() + if state.eager_patched: + uninstall_patches() + uninstall_arch_patches() + # Drop the workflow pipes built around this load's modules so they don't pin the + # freed pipeline (they only re-wire its components, but holding the wrappers + # would keep the modules alive past unload). + self._aux_pipes.clear() self._state = None del state clear_gpu_cache() @@ -930,6 +1453,7 @@ class DiffusionBackend: "base_repo": None, "device": None, "dtype": None, + "model_kind": None, "cpu_offload": False, "offload_policy": None, "vae_tiling": False, @@ -940,6 +1464,7 @@ class DiffusionBackend: "transformer_quant": None, "attention_backend": None, "transformer_cache": None, + "workflows": [], } return { "loaded": True, @@ -948,6 +1473,7 @@ class DiffusionBackend: "base_repo": state.base_repo, "device": state.device, "dtype": state.dtype, + "model_kind": state.kind, "cpu_offload": state.cpu_offload, "offload_policy": state.offload_policy, "vae_tiling": state.vae_tiling, @@ -958,9 +1484,37 @@ class DiffusionBackend: "transformer_quant": state.transformer_quant, "attention_backend": state.attention_backend, "transformer_cache": state.transformer_cache, + # Image-conditioned workflows the loaded family supports, so the UI can gate + # its tabs. txt2img is always available on the diffusers engine. + "workflows": _family_workflows(state.family), } +def _family_workflows(fam: DiffusionFamily) -> list[str]: + """The workflow ids the diffusers engine can run for ``fam`` (drives UI gating).""" + # Instruction-editing families have no plain text-to-image mode: their pipeline always + # takes an input image + instruction, so they expose only the "edit" workflow. + if getattr(fam, "edit", False): + return ["edit"] + workflows = ["txt2img"] + # Reference families (FLUX.2) keep txt2img and add reference conditioning via their own + # pipeline's optional image arg (no img2img/inpaint classes needed). + if getattr(fam, "reference", False): + workflows.append("reference") + if getattr(fam, "img2img_pipeline_class", None): + # Upscale (hires fix) runs on the img2img pipeline, so it is available exactly + # when img2img is. + workflows.append("img2img") + workflows.append("upscale") + if getattr(fam, "inpaint_pipeline_class", None): + workflows.append("inpaint") + # Outpaint (extend) reuses the inpaint pipeline with a padded canvas + border mask, + # so it needs an inpaint pipeline that preserves the (larger) canvas size. + if getattr(fam, "inpaint_preserves_size", True): + workflows.append("outpaint") + return workflows + + def _resolve_base_repo( repo_id: str, base_repo: Optional[str], fam: DiffusionFamily, hf_token: Optional[str] ) -> str: @@ -1004,6 +1558,19 @@ def _base_file_downloaded(rfilename: str) -> bool: return not rfilename.startswith("assets/") +def _pipeline_file_downloaded(rfilename: str) -> bool: + """True for files a full-pipeline ``from_pretrained`` fetches. + + Like ``_base_file_downloaded`` but for the ``pipeline`` kind, where the repo + supplies its OWN transformer weights, so the ``transformer/`` subfolder is kept. + Top-level docs (README/PDF/images) and ``assets/`` are still skipped so the + progress estimate matches what actually lands on disk. + """ + if "/" not in rfilename: # top-level: only the pipeline manifest is fetched + return rfilename == "model_index.json" + return not rfilename.startswith("assets/") + + def _progress( phase: Optional[str], bytes_downloaded: int = 0, diff --git a/studio/backend/core/inference/diffusion_engine_router.py b/studio/backend/core/inference/diffusion_engine_router.py index c8db7f81d2..4dc0d394a7 100644 --- a/studio/backend/core/inference/diffusion_engine_router.py +++ b/studio/backend/core/inference/diffusion_engine_router.py @@ -99,7 +99,9 @@ def _activate(name: str, reason: Optional[str]) -> Any: return get_active_diffusion_engine() -def select_and_activate_engine(fam: DiffusionFamily, *, hf_token: Optional[str] = None) -> Any: +def select_and_activate_engine( + fam: DiffusionFamily, *, hf_token: Optional[str] = None, model_kind: Optional[str] = None +) -> Any: """Pick + activate the engine for loading ``fam`` on this host; return the engine. Falls back to diffusers (recording a reason) whenever the native route is @@ -107,6 +109,12 @@ def select_and_activate_engine(fam: DiffusionFamily, *, hf_token: Optional[str] native asset mapping, or the sd-cli binary is unavailable -- always BEFORE the slow load begins, so a fallback never strands a half-native load. """ + # Non-GGUF loads (a single-file safetensors transformer, or a full diffusers + # pipeline) only run on diffusers: the native sd.cpp engine consumes single-file + # GGUF checkpoints only, so force diffusers before the device/native checks below. + if model_kind and model_kind != "gguf": + return _activate(ENGINE_DIFFUSERS, f"non-GGUF load ({model_kind}) requires diffusers") + forced, sd_cpp_pref, mps_enabled = _engine_config() if forced == ENGINE_DIFFUSERS: diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.py index df3246e2b0..809646d77a 100644 --- a/studio/backend/core/inference/diffusion_families.py +++ b/studio/backend/core/inference/diffusion_families.py @@ -28,6 +28,32 @@ class DiffusionFamily: # Pipeline kwarg carrying the guidance value. Most use "guidance_scale"; # Qwen-Image's distilled guidance is off, so its real CFG is "true_cfg_scale". cfg_kwarg: str = "guidance_scale" + # Optional diffusers pipeline classes for image-conditioned workflows. The backend + # builds these around the ALREADY-loaded transformer/VAE/text-encoder via + # ``Pipeline.from_pipe`` (no extra weights, no reload), so a family only needs the + # class name here to gain the workflow. None = the family does not support it (the + # UI gates the workflow off). The base text-to-image pipeline is ``pipeline_class``. + img2img_pipeline_class: Optional[str] = None + inpaint_pipeline_class: Optional[str] = None + # True when the inpaint pipeline keeps the input canvas size, so it can also drive + # outpaint (extend), where the padded canvas is LARGER than the original. False for + # FLUX.2 (its pipelines scale any >1MP input down to ~1MP, which shrinks an outpaint + # canvas back and defeats the extend). Such families get Inpaint but not Extend. + inpaint_preserves_size: bool = True + # True for instruction-editing families (Qwen-Image-Edit / FLUX Kontext): the model's + # OWN pipeline (``pipeline_class``) is the edit pipeline -- it takes an input image plus + # a text instruction and has no plain text-to-image mode. So these expose only the + # "edit" workflow, require an input image at generate time, and the loaded pipe is used + # directly (no from_pipe). ``base_repo`` here is the matching diffusers repo that + # supplies the VAE / text-encoder / processor / scheduler for the GGUF transformer. + edit: bool = False + # True for families whose OWN text-to-image pipeline ALSO accepts reference image(s) + # (FLUX.2: Flux2KleinPipeline takes an optional ``image`` arg). Unlike ``edit`` these + # families still do plain text-to-image (no image), and unlike img2img the conditioning + # is reference-based, not a denoise blend: there is no ``strength`` and the output size + # comes from the requested width/height, not the reference's size. The loaded pipe is + # used directly (no from_pipe). Exposes a "reference" workflow alongside "txt2img". + reference: bool = False # Extra lowercased substrings (besides ``name``) that map a repo id here. aliases: tuple[str, ...] = field(default_factory = tuple) # True for families whose activations overflow float16's finite range @@ -79,6 +105,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = ( transformer_class = "FluxTransformer2DModel", base_repo = "black-forest-labs/FLUX.1-schnell", aliases = ("flux1", "flux-1"), + img2img_pipeline_class = "FluxImg2ImgPipeline", + inpaint_pipeline_class = "FluxInpaintPipeline", sd_cpp_vae = ("black-forest-labs/FLUX.1-schnell", "ae.safetensors"), sd_cpp_text_encoders = ( ("comfyanonymous/flux_text_encoders", "clip_l.safetensors", "clip_l"), @@ -94,6 +122,13 @@ _FAMILIES: tuple[DiffusionFamily, ...] = ( transformer_class = "Flux2Transformer2DModel", base_repo = "black-forest-labs/FLUX.2-klein-4B", aliases = ("flux2-klein",), + # Flux2KleinPipeline natively accepts reference image(s) via its `image` arg, so it + # exposes a "reference" workflow on top of plain text-to-image. It has a dedicated + # inpaint pipeline too (no img2img one), so it also gets inpaint + extend (outpaint). + reference = True, + inpaint_pipeline_class = "Flux2KleinInpaintPipeline", + # FLUX.2 scales >1MP inputs down to ~1MP, so outpaint (a larger canvas) can't grow. + inpaint_preserves_size = False, # FLUX.2 uses a distinct 32-channel autoencoder; sd-cli needs the latent # format override. The single-file VAE ships in Comfy-Org/flux2-dev (the # klein-4B repo only has a sharded diffusers VAE). Shares Qwen3-4B with z-image. @@ -103,6 +138,39 @@ _FAMILIES: tuple[DiffusionFamily, ...] = ( ("Comfy-Org/z_image_turbo", "split_files/text_encoders/qwen_3_4b.safetensors", "llm"), ), ), + DiffusionFamily( + # Instruction editing with FLUX. FluxKontextPipeline takes an input image + an edit + # instruction; the GGUF transformer is the standard FluxTransformer2DModel, with the + # T5/CLIP text encoders + VAE from the base diffusers repo. cfg defaults to + # guidance_scale (FLUX). Most-specific aliases first so detect_family prefers this + # over the plain "flux.1" family and un-rejects the "kontext" keyword for it. + name = "flux.1-kontext", + pipeline_class = "FluxKontextPipeline", + transformer_class = "FluxTransformer2DModel", + base_repo = "black-forest-labs/FLUX.1-Kontext-dev", + aliases = ("flux.1-kontext-dev", "flux1-kontext", "flux-kontext", "kontext"), + edit = True, + ), + DiffusionFamily( + # Instruction editing (image-in + text-instruction-out). The 2511 checkpoint ships + # as QwenImageEditPlusPipeline (multi-image-capable); the GGUF transformer is the + # standard QwenImageTransformer2DModel, with the VAE / Qwen2.5-VL text-encoder / + # image processor / scheduler coming from the base diffusers repo. Most-specific + # aliases first so detect_family prefers this over the plain "qwen-image" family. + name = "qwen-image-edit", + pipeline_class = "QwenImageEditPlusPipeline", + transformer_class = "QwenImageTransformer2DModel", + base_repo = "Qwen/Qwen-Image-Edit-2511", + cfg_kwarg = "true_cfg_scale", + aliases = ( + "qwen-image-edit-2511", + "qwen-image-edit-2509", + "qwen-image-edit", + "qwen_image_edit", + "qwenimageedit", + ), + edit = True, + ), DiffusionFamily( name = "qwen-image", pipeline_class = "QwenImagePipeline", @@ -110,6 +178,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = ( base_repo = "Qwen/Qwen-Image", cfg_kwarg = "true_cfg_scale", aliases = ("qwen_image", "qwenimage"), + img2img_pipeline_class = "QwenImageImg2ImgPipeline", + inpaint_pipeline_class = "QwenImageInpaintPipeline", sd_cpp_vae = ("Comfy-Org/Qwen-Image_ComfyUI", "split_files/vae/qwen_image_vae.safetensors"), # The Qwen2.5-VL text encoder as a Q4_K_M GGUF keeps the CPU RAM win (the # bf16 safetensors encoder is ~15 GB). sd-cli's --qwen2vl is an alias of --llm. @@ -130,6 +200,8 @@ _FAMILIES: tuple[DiffusionFamily, ...] = ( transformer_class = "ZImageTransformer2DModel", base_repo = "Tongyi-MAI/Z-Image-Turbo", aliases = ("zimage", "z_image"), + img2img_pipeline_class = "ZImageImg2ImgPipeline", + inpaint_pipeline_class = "ZImageInpaintPipeline", # Z-Image's MLP down-projections peak near 9e5, which overflows float16. fp16_incompatible = True, sd_cpp_vae = ("Comfy-Org/z_image_turbo", "split_files/vae/ae.safetensors"), @@ -141,15 +213,33 @@ _FAMILIES: tuple[DiffusionFamily, ...] = ( # Editing / inpaint checkpoints share an arch keyword but need a different # pipeline and an input image, which this text-to-image backend doesn't drive. -_EDIT_KEYWORDS = ("edit", "kontext", "inpaint") +# "layered" rejects Qwen-Image-Layered: its transformer sets additional_t_cond=True +# and expects an extra addition_t_cond input that the standard QwenImagePipeline +# never supplies, so it loads but crashes at the first denoise step. Rejecting it +# here fails the load fast with a clear message and hides it from the picker. +_EDIT_KEYWORDS = ("edit", "kontext", "inpaint", "layered") + + +def _best_family_match(needle: str) -> Optional[DiffusionFamily]: + """The family whose name/alias is the LONGEST substring of ``needle``. Longest = + most specific, so an edit checkpoint ('...qwen-image-edit-2511...') matches the + 'qwen-image-edit' family rather than the generic 'qwen-image' one.""" + best: Optional[tuple[DiffusionFamily, int]] = None + for fam in _FAMILIES: + for token in (fam.name, *fam.aliases): + if token in needle and (best is None or len(token) > best[1]): + best = (fam, len(token)) + return best[0] if best else None def detect_family(repo_id: str, override: Optional[str] = None) -> Optional[DiffusionFamily]: """Resolve a ``DiffusionFamily`` from a repo id, or an explicit override. - ``override`` matches a family ``name`` or alias exactly; otherwise the repo - id is scanned for the first family whose name/alias appears in it. Image - editing checkpoints are rejected (None) since this backend is text-to-image. + ``override`` matches a family ``name`` or alias exactly. Otherwise the most-specific + family whose name/alias is a substring of the repo id wins. Supported editing families + (Qwen-Image-Edit) match here; unsupported editing/inpaint/layered checkpoints that only + share a base family's arch keyword are still rejected (None), because they need a + different pipeline + input this backend's base text-to-image path doesn't drive. """ if override: key = override.strip().lower() @@ -158,11 +248,18 @@ def detect_family(repo_id: str, override: Optional[str] = None) -> Optional[Diff return fam return None needle = repo_id.lower() - if any(kw in needle for kw in _EDIT_KEYWORDS): - return None - for fam in _FAMILIES: - if fam.name in needle or any(alias in needle for alias in fam.aliases): - return fam + match = _best_family_match(needle) + if match is not None: + # Don't let a generic base family (e.g. qwen-image) swallow a variant it can't run + # (qwen-image-LAYERED, ...-Inpaint): if the id still carries a reject keyword the + # matched family does not itself declare, reject so the load fails fast + clearly. + matched_tokens = (match.name, *match.aliases) + if any( + kw in needle and not any(kw in tok for tok in matched_tokens) + for kw in _EDIT_KEYWORDS + ): + return None + return match return None diff --git a/studio/backend/core/inference/diffusion_memory.py b/studio/backend/core/inference/diffusion_memory.py index e896e538d6..cd21d5c72d 100644 --- a/studio/backend/core/inference/diffusion_memory.py +++ b/studio/backend/core/inference/diffusion_memory.py @@ -262,6 +262,17 @@ def estimate_gguf_dense_mib(storage_mib: Optional[int], quant: Optional[str]) -> return int(storage_mib * 4.0) # unknown: assume 4-bit-ish +def estimate_safetensors_dense_mib(storage_mib: Optional[int]) -> Optional[int]: + """Resident size of a safetensors checkpoint, in MiB. + + Unlike a GGUF (which is dequantised to bf16/fp16 on load, so a 4-bit file + expands ~4x), a safetensors checkpoint loads near its on-disk size: a dense + bf16 file is already bf16, and a bnb-4bit / fp8 file stays compressed in VRAM. + So the on-disk size is the estimate, returned unchanged (None passes through). + """ + return storage_mib + + def estimate_image_runtime_mib( *, width: Optional[int], diff --git a/studio/backend/core/inference/sd_cpp_backend.py b/studio/backend/core/inference/sd_cpp_backend.py index 4d0bbcbfa7..0905779787 100644 --- a/studio/backend/core/inference/sd_cpp_backend.py +++ b/studio/backend/core/inference/sd_cpp_backend.py @@ -232,6 +232,9 @@ class SdCppDiffusionBackend: attention_backend: Optional[str] = None, transformer_cache: Optional[str] = None, transformer_cache_threshold: Optional[float] = None, + # Accepted for a uniform engine interface; the native engine is GGUF-only, so a + # non-GGUF kind never routes here (the router forces diffusers for those). + model_kind: Optional[str] = None, ) -> dict[str, Any]: """Validate, then fetch assets on a daemon thread. Returns at once.""" # An empty / whitespace token is "no token": passing "" verbatim to HfApi / @@ -454,11 +457,28 @@ class SdCppDiffusionBackend: guidance: float = 0.0, seed: Optional[int] = None, batch_size: int = 1, + # Accepted for a uniform engine interface. The native engine is text-to-image + # only for now (sd-cli's init-img/mask plumbing is not wired), so an image- + # conditioned request is rejected clearly rather than silently dropping the input. + init_image: Optional[str] = None, + mask_image: Optional[str] = None, + strength: Optional[float] = None, + # Accepted for the uniform engine interface; upscale needs an init image, so the + # init_image guard below rejects it on the native engine like img2img/inpaint. + upscale: Optional[float] = None, + # Reference workflow is GPU/diffusers-only (FLUX.2); accepted for interface parity. + reference_images: Optional[list[str]] = None, ) -> dict[str, Any]: import tempfile from PIL import Image + if init_image is not None or mask_image is not None or reference_images: + raise ValueError( + "img2img / inpaint / reference are not yet supported on the native sd.cpp " + "engine; run on a GPU (diffusers) for image-conditioned workflows." + ) + cancel = threading.Event() with self._generate_lock: with self._lock: diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index db05fdc6f1..5409e08e55 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -1689,9 +1689,19 @@ class AnthropicMessagesResponse(BaseModel): class DiffusionLoadRequest(BaseModel): """Request to load a local diffusion (text-to-image) checkpoint.""" - model_path: str = Field(..., description = "Diffusion GGUF repo id or local path") - gguf_filename: str = Field( - ..., description = "The chosen single-file GGUF quant inside model_path" + model_path: str = Field(..., description = "Diffusion repo id or local path") + gguf_filename: Optional[str] = Field( + None, + description = "The chosen single-file checkpoint (GGUF or safetensors) inside " + "model_path. Required for the gguf / single_file kinds; omit for a full pipeline.", + ) + model_kind: Optional[Literal["gguf", "single_file", "pipeline"]] = Field( + None, + description = "How to load the model (null = auto-detect from gguf_filename): gguf " + "(single-file GGUF transformer, dequantised on-device), single_file (single-file " + "safetensors transformer, e.g. fp8), or pipeline (a full diffusers repo via " + "from_pretrained, embedded quant auto-applied). Non-GGUF kinds are restricted to " + "unsloth/* repos (or a local path).", ) base_repo: Optional[str] = Field( None, description = "Companion diffusers repo for VAE/text-encoders (default: family base)" @@ -1708,10 +1718,12 @@ class DiffusionLoadRequest(BaseModel): "cut), low_vram (offload every component, lowest VRAM, slower). " "Overrides cpu_offload when set.", ) - speed_mode: Optional[Literal["off", "default", "max"]] = Field( + speed_mode: Optional[Literal["off", "eager", "default", "max"]] = Field( None, description = "Opt-in speed optims (default off -> bit-identical output): " - "default (channels_last + regional torch.compile where eligible), " + "eager (channels_last + cudnn + attention + fused RMSNorm/AdaLayerNorm patches, " + "NO torch.compile -> fast first image, no compile tax), " + "default (also regional torch.compile where eligible), " "max (also TF32 + fused QKV).", ) text_encoder_quant: Optional[Literal["fp8", "nvfp4"]] = Field( @@ -1807,6 +1819,55 @@ class DiffusionGenerateRequest(BaseModel): batch_size: int = Field( 1, ge = 1, le = 32, description = "Images generated in one forward pass (VRAM-heavy)" ) + # Image-conditioned workflows (base64 or data-URL). An init_image alone runs img2img; + # init_image + mask_image runs inpaint. Both require a model family with the matching + # pipeline (img2img/inpaint) or the load is rejected with a clear message. + # Cap each base64 image string so a single request can't buffer a multi-GB payload (the + # decoded dimensions are bounded separately in the backend). ~32 MiB comfortably fits a + # full 4096px image yet rejects abuse. + init_image: Optional[str] = Field( + None, + max_length = 32 * 1024 * 1024, + description = "Base64/data-URL source image for img2img or inpaint (omit for txt2img)", + ) + mask_image: Optional[str] = Field( + None, + max_length = 32 * 1024 * 1024, + description = "Base64/data-URL mask for inpaint (white = repaint, black = keep). " + "Requires init_image.", + ) + strength: Optional[float] = Field( + None, + ge = 0.0, + le = 1.0, + description = "img2img/inpaint denoise strength: 0 keeps the source, 1 fully " + "redraws it. Ignored for txt2img.", + ) + upscale: Optional[float] = Field( + None, + ge = 1.0, + le = 4.0, + description = "Upscale (hires fix) factor for an init_image: enlarges the source " + "by this multiple and re-denoises at low strength. Requires init_image; " + "ignored for txt2img/inpaint/edit.", + ) + reference_images: Optional[list[str]] = Field( + None, + max_length = 3, + description = "Additional reference images (base64/data-URL) for the FLUX.2 reference " + "workflow, combined with init_image. Up to 3; ignored by other workflows.", + ) + + @field_validator("reference_images") + @classmethod + def _bounded_reference_items(cls, value: Optional[list[str]]) -> Optional[list[str]]: + # Each reference is a base64 image; bound its length like init_image/mask_image so a + # request carrying several references can't buffer a multi-GB payload. + if value is not None: + for item in value: + if len(item) > 32 * 1024 * 1024: + raise ValueError("each reference image must be at most 32 MiB (base64)") + return value @field_validator("width", "height") @classmethod @@ -1880,6 +1941,9 @@ class DiffusionStatusResponse(BaseModel): base_repo: Optional[str] = Field(None, description = "Companion diffusers base repo") device: Optional[str] = Field(None, description = "Device the pipeline is on") dtype: Optional[str] = Field(None, description = "Compute dtype") + model_kind: Optional[str] = Field( + None, description = "Resolved load kind: gguf | single_file | pipeline (gates GGUF-only UI)" + ) cpu_offload: bool = Field(False, description = "Whether CPU offload is engaged") offload_policy: Optional[str] = Field( None, description = "Resolved offload policy: none | group | model | sequential" @@ -1904,6 +1968,11 @@ class DiffusionStatusResponse(BaseModel): "_native_cudnn), or null for the default SDPA", ) transformer_cache: Optional[str] = Field(None, description = "Step cache engaged: fbcache | null") + workflows: list[str] = Field( + default_factory = list, + description = "Image workflows the loaded family supports (drives UI tab gating): " + "txt2img, img2img, inpaint. Empty when nothing is loaded or on the native engine.", + ) engine: Optional[str] = Field(None, description = "Active diffusion engine: diffusers | sd_cpp") fallback_reason: Optional[str] = Field( None, diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 496a406574..13f45dbae4 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -10057,7 +10057,7 @@ async def _openai_passthrough_non_streaming( async def load_diffusion_model( request: DiffusionLoadRequest, current_subject: str = Depends(get_current_subject) ): - from core.inference.diffusion import get_diffusion_backend + from core.inference.diffusion import get_diffusion_backend, resolve_model_kind from core.inference.diffusion_device import resolve_diffusion_device_target from core.inference.diffusion_engine_router import ( active_engine_name, @@ -10070,19 +10070,25 @@ async def load_diffusion_model( backend = get_diffusion_backend() try: + # Resolve the load kind once (gguf / single_file / pipeline) so validation, + # engine selection, and the load all agree. A bad explicit kind raises here -> 400. + kind = resolve_model_kind(request.gguf_filename, request.model_kind) # Validate cheaply BEFORE touching the GPU: an unloadable pick (bad family, - # missing local GGUF) must not evict a working chat model and then 400. The - # validated family also drives engine selection below. + # missing local GGUF, a non-unsloth non-GGUF repo) must not evict a working chat + # model and then 400. The validated family also drives engine selection below. fam = await asyncio.to_thread( backend.validate_load_request, request.model_path, gguf_filename = request.gguf_filename, family_override = request.family_override, + model_kind = kind, ) # Pick the engine for this host (diffusers on GPU, native sd.cpp with no GPU), # installing the sd-cli binary if needed -- all BEFORE evicting chat, so a - # native fallback never strands a half-loaded state. - engine = await asyncio.to_thread(select_and_activate_engine, fam, hf_token = request.hf_token) + # native fallback never strands a half-loaded state. Non-GGUF kinds force diffusers. + engine = await asyncio.to_thread( + select_and_activate_engine, fam, hf_token = request.hf_token, model_kind = kind + ) # Take the GPU from the chat backend only when this load will actually use it. # diffusers always does; a *force-native* sd.cpp load on a CUDA/XPU/MPS box does # too. But a native sd.cpp load on a pure-CPU host never touches the GPU, so @@ -10110,6 +10116,7 @@ async def load_diffusion_model( attention_backend = request.attention_backend, transformer_cache = request.transformer_cache, transformer_cache_threshold = request.transformer_cache_threshold, + model_kind = kind, ) return DiffusionStatusResponse(**annotate_status(status_dict)) except (ValueError, FileNotFoundError) as exc: @@ -10138,7 +10145,16 @@ async def generate_diffusion_image( guidance = request.guidance, seed = request.seed, batch_size = request.batch_size, + init_image = request.init_image, + mask_image = request.mask_image, + strength = request.strength, + upscale = request.upscale, + reference_images = request.reference_images, ) + except ValueError as exc: + # Bad client input (undecodable image/mask, or a workflow the loaded family + # doesn't support) — a 400 with the reason, not a generic 500. + raise HTTPException(status_code = 400, detail = str(exc)) except RuntimeError as exc: # Only "no model loaded" / cancelled are client-state (409). The native # sd.cpp engine also raises RuntimeError for execution failures (nonzero @@ -10146,10 +10162,10 @@ async def generate_diffusion_image( msg = str(exc) if "No diffusion model is loaded" in msg or "cancelled" in msg.lower(): raise HTTPException(status_code = 409, detail = msg) - logger.error("diffusion.generate_failed: %s", exc) + logger.error("diffusion.generate_failed: %s", exc, exc_info = True) raise HTTPException(status_code = 500, detail = "Image generation failed.") except Exception as exc: - logger.error("diffusion.generate_failed: %s", exc) + logger.error("diffusion.generate_failed: %s", exc, exc_info = True) raise HTTPException(status_code = 500, detail = "Image generation failed.") # Persist each image with its full recipe embedded. The diffusers batch shares diff --git a/studio/backend/tests/test_diffusion_backend.py b/studio/backend/tests/test_diffusion_backend.py index e66f44d100..c988ad4428 100644 --- a/studio/backend/tests/test_diffusion_backend.py +++ b/studio/backend/tests/test_diffusion_backend.py @@ -50,9 +50,24 @@ def test_detect_family_from_repo_id(): # Qwen-Image guides via true_cfg_scale, not guidance_scale. assert detect_family("unsloth/Qwen-Image-2512-GGUF").cfg_kwarg == "true_cfg_scale" assert detect_family("unsloth/Z-Image-GGUF").cfg_kwarg == "guidance_scale" - # Image-editing checkpoints are rejected (text-to-image backend only). - assert detect_family("unsloth/Qwen-Image-Edit-2511-GGUF") is None - assert detect_family("unsloth/FLUX.1-Kontext-dev-GGUF") is None + # Qwen-Image-Edit is a SUPPORTED instruction-editing family (its own edit pipeline); + # the most-specific match wins so it doesn't fall back to the generic qwen-image. + edit = detect_family("unsloth/Qwen-Image-Edit-2511-GGUF") + assert edit.name == "qwen-image-edit" + assert edit.pipeline_class == "QwenImageEditPlusPipeline" + assert edit.edit is True + assert detect_family("unsloth/Qwen-Image-Edit-2509-GGUF").name == "qwen-image-edit" + # FLUX Kontext is a SUPPORTED editing family (FluxKontextPipeline); the "kontext" + # keyword is un-rejected for it, and it must win over the generic "flux.1" match. + kontext = detect_family("unsloth/FLUX.1-Kontext-dev-GGUF") + assert kontext.name == "flux.1-kontext" + assert kontext.pipeline_class == "FluxKontextPipeline" + assert kontext.edit is True + assert kontext.cfg_kwarg == "guidance_scale" + # A plain FLUX.1 checkpoint must still resolve to the base flux.1 family, not kontext. + assert detect_family("unsloth/FLUX.1-dev-GGUF").name == "flux.1" + # A plain Qwen-Image checkpoint must still resolve to the base family, not edit. + assert detect_family("unsloth/Qwen-Image-2512-GGUF").name == "qwen-image" assert detect_family("meta-llama/Llama-3-8B") is None @@ -194,6 +209,86 @@ class _FakeTransformer: return object() +class _FakeImg2ImgPipe: + """An img2img pipeline call: records the image-conditioned kwargs. Its signature + declares image/strength but NOT width/height, mirroring real img2img pipelines + (which derive the output size from the input image).""" + + last_kwargs: dict = {} + + def __call__( + self, + *, + prompt = None, + image = None, + strength = None, + negative_prompt = None, + callback_on_step_end = None, + guidance_scale = None, + true_cfg_scale = None, + **kwargs, + ): + _FakeImg2ImgPipe.last_kwargs = { + "prompt": prompt, + "image": image, + "strength": strength, + **kwargs, + } + n = kwargs.get("num_images_per_prompt", 1) + return types.SimpleNamespace(images = [_FakeImage() for _ in range(n)]) + + +class _FakeImg2ImgPipeline: + built_from: object = None + from_pipe_kwargs: dict = {} + + @classmethod + def from_pipe(cls, base_pipe, **kwargs): + _FakeImg2ImgPipeline.built_from = base_pipe + _FakeImg2ImgPipeline.from_pipe_kwargs = kwargs + return _FakeImg2ImgPipe() + + +class _FakeInpaintPipe: + """An inpaint pipeline call: records image + mask_image + strength. Real inpaint + pipelines take both an init image and a grayscale mask and derive output size from + the input, so width/height are not in its signature.""" + + last_kwargs: dict = {} + + def __call__( + self, + *, + prompt = None, + image = None, + mask_image = None, + strength = None, + negative_prompt = None, + callback_on_step_end = None, + guidance_scale = None, + true_cfg_scale = None, + **kwargs, + ): + _FakeInpaintPipe.last_kwargs = { + "prompt": prompt, + "image": image, + "mask_image": mask_image, + "strength": strength, + **kwargs, + } + n = kwargs.get("num_images_per_prompt", 1) + return types.SimpleNamespace(images = [_FakeImage() for _ in range(n)]) + + +class _FakeInpaintPipeline: + built_from: object = None + + @classmethod + def from_pipe(cls, base_pipe, **kwargs): + _FakeInpaintPipeline.built_from = base_pipe + return _FakeInpaintPipe() + + @pytest.fixture def fake_runtime(monkeypatch): torch = types.ModuleType("torch") @@ -210,9 +305,15 @@ def fake_runtime(monkeypatch): diffusers.GGUFQuantizationConfig = lambda compute_dtype = None: ("quant", compute_dtype) diffusers.ZImagePipeline = _FakePipeline diffusers.ZImageTransformer2DModel = _FakeTransformer + diffusers.ZImageImg2ImgPipeline = _FakeImg2ImgPipeline + diffusers.ZImageInpaintPipeline = _FakeInpaintPipeline # Qwen-Image too, so the true_cfg_scale cfg-kwarg path is exercisable. diffusers.QwenImagePipeline = _FakePipeline diffusers.QwenImageTransformer2DModel = _FakeTransformer + diffusers.QwenImageImg2ImgPipeline = _FakeImg2ImgPipeline + diffusers.QwenImageInpaintPipeline = _FakeInpaintPipeline + # Instruction-editing pipeline (Qwen-Image-Edit): its own pipeline IS the loaded one. + diffusers.QwenImageEditPlusPipeline = _FakePipeline monkeypatch.setitem(sys.modules, "torch", torch) monkeypatch.setitem(sys.modules, "diffusers", diffusers) @@ -221,6 +322,10 @@ def fake_runtime(monkeypatch): monkeypatch.setattr("core.inference.diffusion.clear_gpu_cache", lambda: None) _FakePipeline.last = {} _FakeTransformer.last = {} + _FakeImg2ImgPipeline.built_from = None + _FakeImg2ImgPipe.last_kwargs = {} + _FakeInpaintPipeline.built_from = None + _FakeInpaintPipe.last_kwargs = {} yield @@ -273,6 +378,453 @@ def test_load_generate_unload_gguf(fake_runtime, tmp_path): assert backend.is_loaded is False +def _tiny_png_b64() -> str: + import base64 + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (64, 64), (120, 30, 30)).save(buf, format = "PNG") + return base64.b64encode(buf.getvalue()).decode() + + +def test_generate_img2img_uses_from_pipe(fake_runtime, tmp_path): + """An init_image routes generate() through the family's img2img pipeline, built via + Pipeline.from_pipe around the loaded pipe (no reload), with image + strength passed + and width/height dropped (the img2img pipe derives size from the input image).""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + # The loaded family advertises the image-conditioned workflows for UI gating + # (upscale rides the img2img pipeline, so it appears whenever img2img does). + assert backend.status()["workflows"] == ["txt2img", "img2img", "upscale", "inpaint", "outpaint"] + + loaded_pipe = backend._state.pipe + out = backend.generate( + prompt = "a car at sunset", steps = 4, guidance = 0.0, seed = 3, + init_image = _tiny_png_b64(), strength = 0.5, + ) + assert len(out["images"]) == 1 + # from_pipe was handed the loaded text-to-image pipe (component reuse, no reload). + assert _FakeImg2ImgPipeline.built_from is loaded_pipe + # ...and with torch_dtype=None so from_pipe SKIPS its default float32 recast, which + # both upcasts the reused bf16 modules and crashes on torchao-quantized weights. + assert _FakeImg2ImgPipeline.from_pipe_kwargs.get("torch_dtype", "MISSING") is None + call = _FakeImg2ImgPipe.last_kwargs + assert call["image"] is not None # decoded source image passed through + assert call["strength"] == 0.5 + assert "width" not in call and "height" not in call # img2img derives size from image + + # A txt2img call after it still uses the base pipe (no image kwarg). + backend.generate(prompt = "plain", steps = 4, seed = 1) + assert backend._state.pipe.last_kwargs.get("image") is None + + +def test_generate_img2img_unsupported_family_raises(fake_runtime, tmp_path, monkeypatch): + """A family with no image-conditioning at all (no img2img/inpaint/edit/reference) rejects + an init_image with a clear error rather than failing deep in the pipeline.""" + from core.inference.diffusion_families import DiffusionFamily + + # A synthetic txt2img-only family: no img2img/inpaint pipeline, not edit, not reference. + # (Every shipped family now supports some image workflow, so build one for this case.) + plain = DiffusionFamily( + name = "plain-test", + pipeline_class = "ZImagePipeline", + transformer_class = "ZImageTransformer2DModel", + base_repo = "base/repo", + ) + monkeypatch.setattr( + "core.inference.diffusion.detect_family", lambda repo_id, override = None: plain + ) + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline(str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo") + assert backend.status()["workflows"] == ["txt2img"] + with pytest.raises(ValueError, match = "img2img"): + backend.generate(prompt = "x", steps = 4, init_image = _tiny_png_b64()) + + +def test_generate_upscale_enlarges_and_low_strength(fake_runtime, tmp_path): + """An init_image + upscale factor routes generate() through the family's img2img + pipeline (hires fix): the source is enlarged to size*factor (rounded to /16) before the + denoise, the strength defaults low, and the factor is capped so a huge value can't OOM.""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + # Upscale rides the img2img pipeline, so it is advertised alongside img2img. + assert "upscale" in backend.status()["workflows"] + + loaded_pipe = backend._state.pipe + out = backend.generate( + prompt = "a crisp photo", steps = 4, guidance = 0.0, seed = 3, + init_image = _tiny_png_b64(), upscale = 2.0, # 64 -> 128, no explicit strength + ) + assert len(out["images"]) == 1 + # Reuses the resident modules via from_pipe (no reload, no extra VRAM). + assert _FakeImg2ImgPipeline.built_from is loaded_pipe + call = _FakeImg2ImgPipe.last_kwargs + # The image handed to the pipe is the ENLARGED source (64 * 2 = 128, already /16). + assert call["image"].size == (128, 128) + # Strength defaults to the hires-fix value when the caller sends none. + assert call["strength"] == 0.35 + + # The factor is capped at 4x so a large request can't blow up the VAE/transformer. + backend.generate( + prompt = "x", steps = 4, seed = 1, init_image = _tiny_png_b64(), upscale = 99.0, + ) + assert _FakeImg2ImgPipe.last_kwargs["image"].size == (256, 256) # 64 * 4 (capped) + + # An explicit strength overrides the hires-fix default. + backend.generate( + prompt = "x", steps = 4, seed = 1, init_image = _tiny_png_b64(), + upscale = 1.5, strength = 0.2, + ) + assert _FakeImg2ImgPipe.last_kwargs["strength"] == 0.2 + # 64 * 1.5 = 96, already a multiple of 16. + assert _FakeImg2ImgPipe.last_kwargs["image"].size == (96, 96) + + +def _png_b64(side: int) -> str: + import base64 + import io + + from PIL import Image + + buf = io.BytesIO() + Image.new("RGB", (side, side), (10, 20, 30)).save(buf, format = "PNG") + return base64.b64encode(buf.getvalue()).decode() + + +def test_decode_image_rejects_oversized(fake_runtime, tmp_path): + """An input image larger than the per-side cap is rejected with a clear error (protects + img2img / inpaint / reference from decompression-bomb / OOM inputs), not a 500.""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + with pytest.raises(ValueError, match = "too large"): + backend.generate(prompt = "x", steps = 4, init_image = _png_b64(4112)) # > 4096/side + + +def test_upscale_output_is_capped(fake_runtime, tmp_path): + """Upscale bounds the absolute output side to 2048 even when input*factor exceeds it, so a + large upload at 4x can't OOM the VAE/transformer.""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + backend.generate(prompt = "x", steps = 4, seed = 1, init_image = _png_b64(1024), upscale = 4.0) + # 1024 * 4 = 4096 -> clamped to 2048 (longest side), still a multiple of 16. + assert _FakeImg2ImgPipe.last_kwargs["image"].size == (2048, 2048) + + +def _mask_b64(side: int) -> str: + import base64 + import io + + from PIL import Image + + buf = io.BytesIO() + img = Image.new("L", (side, side), 0) + for y in range(side // 4, 3 * side // 4): + for x in range(side // 4, 3 * side // 4): + img.putpixel((x, y), 255) + img.save(buf, format = "PNG") + return base64.b64encode(buf.getvalue()).decode() + + +def test_img2img_snaps_non_multiple_of_16(fake_runtime, tmp_path): + """An odd-sized img2img upload (not divisible by 16) is auto-resized to the nearest + multiple of 16 so the pipeline's divisibility check passes instead of erroring.""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + backend.generate(prompt = "x", steps = 4, seed = 1, init_image = _png_b64(186), strength = 0.5) + # 186 / 16 = 11.625 -> round to 12 -> 192. + assert _FakeImg2ImgPipe.last_kwargs["image"].size == (192, 192) + + +def test_inpaint_snaps_image_and_mask_together(fake_runtime, tmp_path): + """Inpaint snaps the odd-sized input to /16 AND resizes the mask to match, so the image + and mask stay aligned (a mismatch would crash the inpaint pipeline).""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + backend.generate( + prompt = "x", steps = 4, seed = 1, + init_image = _png_b64(186), mask_image = _mask_b64(186), strength = 0.5, + ) + assert _FakeInpaintPipe.last_kwargs["image"].size == (192, 192) + assert _FakeInpaintPipe.last_kwargs["mask_image"].size == (192, 192) + + +def test_generate_reference_uses_loaded_pipe_at_slider_size(fake_runtime, tmp_path): + """A reference family (FLUX.2-klein) advertises txt2img + reference, and a generate with + an init_image passes it as the loaded pipe's `image` arg (no from_pipe, no strength) while + the output size stays the REQUESTED slider size (the pipe resizes the reference itself).""" + import diffusers + + diffusers.Flux2KleinPipeline = _FakePipeline + diffusers.Flux2KleinInpaintPipeline = _FakeInpaintPipeline + diffusers.Flux2Transformer2DModel = _FakeTransformer + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", + family_override = "flux.2-klein", + ) + # FLUX.2-klein: txt2img + reference (own pipe) + inpaint (dedicated pipe). No img2img class, + # so no img2img/upscale. + assert backend.status()["workflows"] == ["txt2img", "reference", "inpaint"] + + loaded_pipe = backend._state.pipe + out = backend.generate( + prompt = "a portrait in this style", steps = 6, guidance = 4.0, seed = 5, + width = 768, height = 512, init_image = _tiny_png_b64(), strength = 0.5, + ) + assert len(out["images"]) == 1 + call = loaded_pipe.last_kwargs + assert call["image"] is not None # reference handed to the loaded pipe + assert call["width"] == 768 and call["height"] == 512 # OUTPUT size = sliders, not input + assert "strength" not in call # reference conditioning has no strength + assert "mask_image" not in call + # Guidance flows via guidance_scale (FLUX.2 default behaviour). + assert call["guidance_scale"] == 4.0 + + # Multi-reference: extra reference_images are combined with init_image into a LIST so the + # model can blend several references (subject + style). + backend.generate( + prompt = "combine these", steps = 6, seed = 9, width = 1024, height = 1024, + init_image = _tiny_png_b64(), reference_images = [_tiny_png_b64(), _tiny_png_b64()], + ) + img_arg = loaded_pipe.last_kwargs["image"] + assert isinstance(img_arg, list) and len(img_arg) == 3 # primary + 2 extras + + # Branch ordering: an init image + MASK on a reference family must route to inpaint (the + # dedicated pipeline), NOT be swallowed by the reference branch (which ignores the mask). + backend.generate( + prompt = "repaint here", steps = 6, seed = 2, + init_image = _tiny_png_b64(), mask_image = _tiny_mask_b64(), strength = 0.8, + ) + assert _FakeInpaintPipeline.built_from is loaded_pipe # built via from_pipe off the load + assert _FakeInpaintPipe.last_kwargs["mask_image"] is not None + assert _FakeInpaintPipe.last_kwargs["strength"] == 0.8 + + # Without an init image the same family does plain txt2img (no image arg). + backend.generate(prompt = "just text", steps = 6, seed = 1) + assert backend._state.pipe.last_kwargs.get("image") is None + + +def _tiny_mask_b64() -> str: + import base64 + import io + + from PIL import Image + + buf = io.BytesIO() + # A grayscale mask: white square (repaint) on black (keep). + img = Image.new("L", (64, 64), 0) + for y in range(16, 48): + for x in range(16, 48): + img.putpixel((x, y), 255) + img.save(buf, format = "PNG") + return base64.b64encode(buf.getvalue()).decode() + + +def test_generate_inpaint_uses_from_pipe(fake_runtime, tmp_path): + """An init_image + mask_image routes generate() through the family's inpaint pipeline, + built via Pipeline.from_pipe around the loaded pipe (no reload), with the decoded image + + mask + strength passed through and width/height dropped (size derives from the input).""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + loaded_pipe = backend._state.pipe + out = backend.generate( + prompt = "a red door", steps = 4, guidance = 0.0, seed = 5, + init_image = _tiny_png_b64(), mask_image = _tiny_mask_b64(), strength = 0.7, + ) + assert len(out["images"]) == 1 + # The inpaint pipe (not img2img) was selected and built from the loaded pipe. + assert _FakeInpaintPipeline.built_from is loaded_pipe + assert _FakeImg2ImgPipeline.built_from is None + call = _FakeInpaintPipe.last_kwargs + assert call["image"] is not None and call["mask_image"] is not None + assert call["strength"] == 0.7 + assert "width" not in call and "height" not in call # inpaint derives size from image + + +def test_image_conditioned_passes_image_size_not_slider(fake_runtime, tmp_path): + """When the workflow pipe DOES accept width/height, an image-conditioned call must pass + the INPUT IMAGE's size, never the txt2img slider size -- otherwise a non-slider-sized + input (e.g. a 1536px outpaint canvas with a 1024 slider) mismatches the latents + ("tensor a (128) must match tensor b (192)"). Covers Transform + Extend with any size.""" + import base64 + import io + + from PIL import Image + + class _SizePipe: + last: dict = {} + + def __call__( + self, + *, + prompt = None, + image = None, + strength = None, + width = None, + height = None, + negative_prompt = None, + callback_on_step_end = None, + guidance_scale = None, + true_cfg_scale = None, + **kwargs, + ): + _SizePipe.last = {"width": width, "height": height} + n = kwargs.get("num_images_per_prompt", 1) + return types.SimpleNamespace(images = [_FakeImage() for _ in range(n)]) + + class _SizePipeline: + @classmethod + def from_pipe(cls, base_pipe, **kwargs): + return _SizePipe() + + import diffusers + + diffusers.ZImageImg2ImgPipeline = _SizePipeline + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "base/repo", family_override = "z-image" + ) + buf = io.BytesIO() + Image.new("RGB", (96, 64), (10, 20, 30)).save(buf, format = "PNG") # non-square, non-slider + b64 = base64.b64encode(buf.getvalue()).decode() + backend.generate(prompt = "x", steps = 4, width = 1024, height = 1024, init_image = b64, strength = 0.5) + # The pipe got the IMAGE's 96x64, not the 1024x1024 slider. + assert _SizePipe.last == {"width": 96, "height": 64} + + +def test_edit_family_uses_own_pipeline_and_requires_image(fake_runtime, tmp_path): + """An instruction-editing family (Qwen-Image-Edit) exposes only the 'edit' workflow, + runs the image through its OWN loaded pipeline (no from_pipe), and rejects a call with + no input image.""" + (tmp_path / "model.gguf").write_bytes(b"x") + backend = DiffusionBackend() + backend.load_pipeline( + str(tmp_path), gguf_filename = "model.gguf", base_repo = "Qwen/Qwen-Image-Edit-2511", + family_override = "qwen-image-edit", + ) + # Edit families advertise only the edit workflow (no txt2img / img2img / inpaint). + assert backend.status()["workflows"] == ["edit"] + loaded_pipe = backend._state.pipe + + out = backend.generate( + prompt = "make it night", steps = 8, guidance = 4.0, seed = 1, + init_image = _tiny_png_b64(), + ) + assert len(out["images"]) == 1 + # The loaded pipe handled it directly -- no from_pipe img2img/inpaint was built. + assert backend._state.pipe is loaded_pipe + assert _FakeImg2ImgPipeline.built_from is None and _FakeInpaintPipeline.built_from is None + assert loaded_pipe.last_kwargs.get("image") is not None + + # An edit model with no input image fails fast with a clear message. + with pytest.raises(ValueError, match = "image"): + backend.generate(prompt = "make it night", steps = 8) + + +def test_load_pipeline_kind_uses_from_pretrained(fake_runtime): + """A full-pipeline (no single-file) load on an unsloth/* repo builds the pipe with + pipeline_cls.from_pretrained(repo_id) -- NO single-file transformer build, NO GGUF + quant config -- so an embedded bnb-4bit config is reloaded by diffusers itself.""" + backend = DiffusionBackend() + status = backend.load_pipeline( + "unsloth/Z-Image-Turbo-unsloth-bnb-4bit", family_override = "z-image" + ) + assert status["loaded"] is True + assert status["family"] == "z-image" + # from_pretrained pointed at the repo itself (it IS its own base), with no transformer. + assert _FakePipeline.last["base"] == "unsloth/Z-Image-Turbo-unsloth-bnb-4bit" + assert "transformer" not in _FakePipeline.last + # The GGUF single-file build path was never taken. + assert _FakeTransformer.last == {} + + +def test_load_single_file_safetensors_no_gguf_config(fake_runtime, tmp_path): + """A single-file *.safetensors transformer is built with from_single_file WITHOUT the + GGUF dequant config (it carries its own dtype), then assembled from the base repo.""" + (tmp_path / "model.safetensors").write_bytes(b"weights") + backend = DiffusionBackend() + status = backend.load_pipeline( + str(tmp_path), + gguf_filename = "model.safetensors", + base_repo = "base/repo", + family_override = "qwen-image", + ) + assert status["loaded"] is True + assert _FakeTransformer.last["path"] == str((tmp_path / "model.safetensors").resolve()) + assert _FakeTransformer.last["subfolder"] == "transformer" + # No GGUF quant config on the safetensors path (the GGUF path sets one). + assert "quantization_config" not in _FakeTransformer.last + assert _FakePipeline.last["base"] == "base/repo" + assert "transformer" in _FakePipeline.last + + +def test_load_pipeline_rejects_non_unsloth_repo(fake_runtime): + backend = DiffusionBackend() + with pytest.raises(ValueError, match = "unsloth"): + backend.load_pipeline("randomorg/Z-Image-bnb-4bit", family_override = "z-image") + + +def test_detect_family_rejects_layered(): + # Qwen-Image-Layered needs a dedicated pipeline (additional_t_cond); it must be + # rejected so it fails fast at load instead of crashing at the first denoise step. + assert detect_family("unsloth/Qwen-Image-Layered-GGUF") is None + assert detect_family("unsloth/qwen_image_layered") is None + + +def test_failed_load_rolls_back_eager_patches(fake_runtime, tmp_path, monkeypatch): + """A load failure AFTER the eager patches install but BEFORE the _LoadState commit must + roll the process-wide patches back, so the next bit-identical `off` load is not + contaminated (the asymmetric-cleanup bug the reviewers flagged).""" + from core.inference import diffusion as diff_mod + from core.inference import diffusion_eager_patches as ep + + (tmp_path / "model.gguf").write_bytes(b"x") + ep.uninstall_patches() # clean slate + + def _boom(*_a, **_k): + raise RuntimeError("placement boom") + + # apply_memory_plan runs AFTER the patches are installed, before _LoadState commits. + monkeypatch.setattr(diff_mod, "apply_memory_plan", _boom) + backend = DiffusionBackend() + with pytest.raises(RuntimeError): + backend.load_pipeline( + str(tmp_path), + gguf_filename = "model.gguf", + family_override = "z-image", + base_repo = "base/repo", + speed_mode = "eager", # != off -> installs the shared patches + ) + assert ep.is_installed() is False # rolled back by the load-failure finally + assert backend.is_loaded is False + + def test_cpu_offload_ignored_off_cuda(fake_runtime, tmp_path): (tmp_path / "model.gguf").write_bytes(b"x") backend = DiffusionBackend() @@ -319,8 +871,10 @@ def test_resolve_base_repo_prefers_caller_then_hf_tag_then_fallback(monkeypatch) def test_load_without_gguf_raises(): backend = DiffusionBackend() - with pytest.raises(ValueError): - backend.load_pipeline("unsloth/Z-Image-Turbo-GGUF") # no gguf_filename + # No gguf_filename -> a full-pipeline load, gated to unsloth/*; a non-unsloth repo + # is rejected before any GPU/network work. + with pytest.raises(ValueError, match = "unsloth"): + backend.load_pipeline("some-org/Z-Image-bnb-4bit") def test_load_unknown_family_raises(): @@ -685,8 +1239,24 @@ def test_callback_cancellation_interrupts_denoise(fake_runtime): def test_validate_load_request(tmp_path): backend = DiffusionBackend() - with pytest.raises(ValueError, match = "gguf_filename"): - backend.validate_load_request("unsloth/Z-Image-Turbo-GGUF") + # No filename + unsloth repo -> a full-pipeline load (allowed for unsloth/*). + assert ( + backend.validate_load_request("unsloth/Z-Image-Turbo-unsloth-bnb-4bit").name == "z-image" + ) + # No filename + non-unsloth repo -> a pipeline load, gated to unsloth/* -> rejected. + with pytest.raises(ValueError, match = "unsloth"): + backend.validate_load_request("some-org/Z-Image-bnb-4bit") + # An explicit gguf/single_file kind still requires a single-file name. + with pytest.raises(ValueError, match = "single-file"): + backend.validate_load_request("unsloth/Z-Image-Turbo-GGUF", model_kind = "gguf") + # A pipeline kind must NOT carry a single-file name. + with pytest.raises(ValueError, match = "pipeline"): + backend.validate_load_request( + "unsloth/Z-Image-Turbo-bnb-4bit", gguf_filename = "q.gguf", model_kind = "pipeline" + ) + # A single-file safetensors load is also gated to unsloth/* repos. + with pytest.raises(ValueError, match = "unsloth"): + backend.validate_load_request("some-org/Z-Image", gguf_filename = "model.safetensors") with pytest.raises(ValueError, match = "family"): backend.validate_load_request("meta/Llama-3", gguf_filename = "q.gguf") assert ( diff --git a/studio/backend/tests/test_diffusion_routes.py b/studio/backend/tests/test_diffusion_routes.py index 4fc9d66c34..c2102adebe 100644 --- a/studio/backend/tests/test_diffusion_routes.py +++ b/studio/backend/tests/test_diffusion_routes.py @@ -35,13 +35,21 @@ class _FakeBackend: *, gguf_filename = None, family_override = None, + model_kind = None, ): # Mirror the real backend's cheap validation so the route's # validate-before-evict ordering is exercised. + from core.inference.diffusion import resolve_model_kind from core.inference.diffusion_families import detect_family - if not gguf_filename: - raise ValueError("gguf_filename is required.") + kind = resolve_model_kind(gguf_filename, model_kind) + if kind in ("gguf", "single_file") and not gguf_filename: + raise ValueError("a single-file checkpoint name is required.") + # Non-GGUF loads are gated to unsloth/* (or a local path), like the real backend. + if kind != "gguf" and not model_path.lower().startswith("unsloth/"): + raise ValueError( + f"Non-GGUF diffusion loads are restricted to unsloth/* repos; got '{model_path}'." + ) fam = detect_family(model_path, family_override) if fam is None: raise ValueError(f"Could not infer a diffusion family for '{model_path}'.") @@ -254,10 +262,24 @@ def test_generate_rejects_non_multiple_of_16(client): assert ok.status_code == 200 -def test_load_requires_gguf_filename(client): - # gguf_filename is now mandatory — a load without it is a 422. +def test_non_gguf_load_restricted_to_unsloth(client): + # gguf_filename is optional now; with none, the load is a full-pipeline kind, which + # is gated to unsloth/* repos. A non-unsloth repo (no filename) is rejected -> 400. resp = client.post("/api/inference/images/load", json = {"model_path": "x/z-image"}) - assert resp.status_code == 422 + assert resp.status_code == 400 + assert "unsloth" in resp.json()["detail"].lower() + + +def test_pipeline_load_allowed_for_unsloth_repo(client): + # An unsloth/* repo with no filename loads as a full diffusers pipeline (kind auto + # = pipeline); the route forwards model_kind="pipeline" to begin_load. + resp = client.post( + "/api/inference/images/load", json = {"model_path": "unsloth/Z-Image-Turbo-unsloth-bnb-4bit"} + ) + assert resp.status_code == 200 + backend = diffusion_module.get_diffusion_backend() + assert backend.last_load_kwargs["model_kind"] == "pipeline" + assert backend.last_load_kwargs.get("gguf_filename") is None def test_generate_without_load_returns_409(client): diff --git a/studio/backend/tests/test_sd_cpp_install.py b/studio/backend/tests/test_sd_cpp_install.py index 171ea3e810..86fa839882 100644 --- a/studio/backend/tests/test_sd_cpp_install.py +++ b/studio/backend/tests/test_sd_cpp_install.py @@ -17,7 +17,26 @@ _STUDIO = Path(__file__).resolve().parents[2] if str(_STUDIO) not in sys.path: sys.path.insert(0, str(_STUDIO)) -from install_sd_cpp_prebuilt import default_install_dir, resolve_release_asset # noqa: E402 +import hashlib # noqa: E402 +import io # noqa: E402 +import json # noqa: E402 +import urllib.error # noqa: E402 +import zipfile # noqa: E402 + +import pytest # noqa: E402 + +import install_sd_cpp_prebuilt as sdmod # noqa: E402 +from install_sd_cpp_prebuilt import ( # noqa: E402 + DEFAULT_REPO, + DEFAULT_TAG, + _fetch_release, + _pinned_tag, + _repo, + _verify_sha256, + default_install_dir, + install, + resolve_release_asset, +) # A real stable-diffusion.cpp latest-release asset list. _ASSETS = [ @@ -114,3 +133,127 @@ def test_default_install_dir_is_sibling_of_llama(monkeypatch): d = default_install_dir() assert d.name == "stable-diffusion.cpp" assert d.parent.name == ".unsloth" + + +# ── version pin + source repo (reproducibility) ───────────────────────────── + + +def test_pinned_tag_default_and_override(monkeypatch): + monkeypatch.delenv("UNSLOTH_SD_CPP_TAG", raising = False) + assert _pinned_tag() == DEFAULT_TAG # pinned, not "latest" + monkeypatch.setenv("UNSLOTH_SD_CPP_TAG", "master-999-deadbee") + assert _pinned_tag() == "master-999-deadbee" + monkeypatch.setenv("UNSLOTH_SD_CPP_TAG", "") # explicit empty -> track latest + assert _pinned_tag() is None + + +def test_repo_default_and_mirror_override(monkeypatch): + monkeypatch.delenv("UNSLOTH_SD_CPP_REPO", raising = False) + assert _repo() == DEFAULT_REPO == "leejet/stable-diffusion.cpp" + monkeypatch.setenv("UNSLOTH_SD_CPP_REPO", "unslothai/stable-diffusion.cpp") + assert _repo() == "unslothai/stable-diffusion.cpp" + + +# ── sha256 integrity check ────────────────────────────────────────────────── + + +def test_verify_sha256_accepts_matching_digest(tmp_path): + f = tmp_path / "asset.zip" + f.write_bytes(b"hello sd-cli") + digest = "sha256:" + hashlib.sha256(b"hello sd-cli").hexdigest() + _verify_sha256(f, digest) # no raise + + +def test_verify_sha256_rejects_mismatch(tmp_path): + f = tmp_path / "asset.zip" + f.write_bytes(b"tampered") + bad = "sha256:" + hashlib.sha256(b"original").hexdigest() + with pytest.raises(RuntimeError, match = "sha256 mismatch"): + _verify_sha256(f, bad) + + +def test_verify_sha256_skips_when_absent_or_unknown(tmp_path): + f = tmp_path / "asset.zip" + f.write_bytes(b"x") + _verify_sha256(f, None) # no digest published -> warn + proceed (no raise) + _verify_sha256(f, "md5:abc") # unrecognised algo -> skip (no raise) + + +# ── _fetch_release: pinned-tag 404 -> latest fallback ─────────────────────── + + +def test_fetch_release_falls_back_to_latest_on_404(monkeypatch): + calls: list[str] = [] + + class _Resp: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return json.dumps({"tag_name": "latest-xyz", "assets": []}).encode() + + def fake_urlopen(req, timeout = 30.0): + url = getattr(req, "full_url", req) + calls.append(url) + if "/tags/" in url: + raise urllib.error.HTTPError(url, 404, "not found", None, None) + return _Resp() + + monkeypatch.setattr(sdmod.urllib.request, "urlopen", fake_urlopen) + rel = _fetch_release("gone-tag", repo = "leejet/stable-diffusion.cpp") + assert rel["tag_name"] == "latest-xyz" + assert any("/tags/gone-tag" in c for c in calls) and any(c.endswith("/latest") for c in calls) + + +def test_fetch_release_propagates_non_404(monkeypatch): + def fake_urlopen(req, timeout = 30.0): + url = getattr(req, "full_url", req) + raise urllib.error.HTTPError(url, 403, "rate limited", None, None) + + monkeypatch.setattr(sdmod.urllib.request, "urlopen", fake_urlopen) + with pytest.raises(urllib.error.HTTPError): + _fetch_release("any-tag") + + +# ── install(): download -> verify -> extract -> locate (offline) ──────────── + + +def _zip_with_sd_cli() -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("build/bin/sd-cli", b"#!/bin/sh\necho sd-cli\n") + return buf.getvalue() + + +def _stub_release(monkeypatch, *, zip_bytes: bytes, digest: str): + name = "sd-master-deadbee-bin-Linux-Ubuntu-24.04-x86_64.zip" + release = { + "tag_name": "master-1-deadbee", + "assets": [ + {"name": name, "browser_download_url": f"https://example.invalid/{name}", "digest": digest} + ], + } + monkeypatch.setattr(sdmod, "_fetch_release", lambda *a, **k: release) + monkeypatch.setattr(sdmod, "_download", lambda url, dest, **k: dest.write_bytes(zip_bytes)) + monkeypatch.setattr(sdmod.platform, "system", lambda: "Linux") + monkeypatch.setattr(sdmod.platform, "machine", lambda: "x86_64") + return name + + +def test_install_downloads_verifies_extracts(tmp_path, monkeypatch): + zb = _zip_with_sd_cli() + name = _stub_release(monkeypatch, zip_bytes = zb, digest = "sha256:" + hashlib.sha256(zb).hexdigest()) + sd_cli = install(install_dir = tmp_path) + assert sd_cli.name == "sd-cli" and sd_cli.is_file() + assert not (tmp_path / name).exists() # archive cleaned up after extract + + +def test_install_sha256_mismatch_raises_and_cleans_up(tmp_path, monkeypatch): + zb = _zip_with_sd_cli() + name = _stub_release(monkeypatch, zip_bytes = zb, digest = "sha256:" + "0" * 64) + with pytest.raises(RuntimeError, match = "sha256 mismatch"): + install(install_dir = tmp_path) + assert not (tmp_path / name).exists() # the finally: drops the bad archive diff --git a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx index a49acda408..aae3d75bff 100644 --- a/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx +++ b/studio/frontend/src/components/assistant-ui/model-selector/pickers.tsx @@ -989,11 +989,17 @@ export const IMAGE_GEN_TASKS = [ // which the text-to-image backend rejects (mirrors its _EDIT_KEYWORDS). Hidden by // id so they don't show in the Images picker only to 400 on load. Keeping the // image-to-image task itself is required: some supported models (FLUX.2-klein) -// carry that tag too. -const IMAGE_EDIT_KEYWORDS = ["edit", "kontext", "inpaint"] as const; +// carry that tag too. "layered" hides Qwen-Image-Layered, which needs a dedicated +// pipeline (additional_t_cond) the standard text-to-image path can't drive. +const IMAGE_EDIT_KEYWORDS = ["edit", "kontext", "inpaint", "layered"] as const; +// Editing families the backend now SUPPORTS (their own Edit workflow) -- must not be +// hidden even though their id contains an edit keyword. Mirrors the backend's +// qwen-image-edit family in diffusion_families.py. +const SUPPORTED_EDIT_KEYWORDS = ["qwen-image-edit", "kontext"] as const; function isImageEditModel(repoId: string | null | undefined): boolean { if (!repoId) return false; const id = repoId.toLowerCase(); + if (SUPPORTED_EDIT_KEYWORDS.some((kw) => id.includes(kw))) return false; return IMAGE_EDIT_KEYWORDS.some((kw) => id.includes(kw)); } @@ -1676,6 +1682,19 @@ export function HubModelPicker({ isChatSupported, ]); + // Curated non-GGUF (safetensors) models for the Images picker. The HF listing + + // Recommended gate only surface GGUF on a GPU host (isRecommendableFormat), so a + // bnb-4bit / fp8 safetensors model would never appear there. These curated entries + // (the non-GGUF ModelOptions passed in) are shown explicitly above the GGUF rows so + // the user can pick a full diffusers pipeline. Only the Images picker (task set) + // curates them; already-downloaded ones show under Downloaded instead. + const curatedSafetensorsRows = useMemo(() => { + if (!task) return []; + return models.filter( + (m) => m.isGguf === false && !downloadedSet.has(m.id.toLowerCase()), + ); + }, [models, task, downloadedSet]); + // Per-row meta + VRAM badge from the recommended listing's own metadata. const recommendedMeta = useMemo(() => { const map = new Map< @@ -3244,6 +3263,30 @@ export function HubModelPicker({ {showRecommendedSection ? ( <> + {/* Curated safetensors models (full diffusers pipelines / single-file + fp8). Shown above the GGUF rows; clicking loads directly (no quant + expander), the same path as a non-GGUF Recommended row. */} + {curatedSafetensorsRows.map((m) => { + const optionKey = makeModelOptionKey("curated-safetensors", m.id); + return ( +
+ handleModelClick(m.id)} + gpuGb={gpu.available ? gpu.memoryTotalGb : undefined} + /> +
+ ); + })} {recommendedSearch.isLoading && recommendedRows.length === 0 ? (
@@ -3252,7 +3295,8 @@ export function HubModelPicker({ Loading models…
- ) : recommendedRows.length === 0 ? ( + ) : recommendedRows.length === 0 && + curatedSafetensorsRows.length === 0 ? (
No models found.
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts index fb9236cb4b..3bb3c92518 100644 --- a/studio/frontend/src/features/images/api.ts +++ b/studio/frontend/src/features/images/api.ts @@ -11,7 +11,13 @@ export interface DiffusionStatus { base_repo: string | null; device: string | null; dtype: string | null; + // Resolved load kind: "gguf" | "single_file" | "pipeline". Gates GGUF-only controls + // (the dense transformer_quant fast path only engages on gguf). Null when not loaded. + model_kind?: string | null; cpu_offload: boolean; + // Image workflows the loaded family supports (drives tab gating): txt2img, img2img, + // inpaint. Absent/empty when nothing is loaded or on the native sd.cpp engine. + workflows?: string[]; } export interface DiffusionGenerateProgress { @@ -32,11 +38,33 @@ export interface DiffusionLoadProgress { export interface DiffusionLoadRequest { model_path: string; - gguf_filename: string; + // Optional now: required for the gguf / single_file kinds, omitted for a full + // pipeline (a diffusers repo loaded via from_pretrained). + gguf_filename?: string; + // How to load the model (omit to auto-detect from gguf_filename): "gguf" (single-file + // GGUF transformer), "single_file" (single-file safetensors transformer, e.g. fp8), or + // "pipeline" (a full diffusers repo). Non-GGUF kinds are restricted to unsloth/* repos. + model_kind?: "gguf" | "single_file" | "pipeline"; base_repo?: string; family_override?: string; hf_token?: string; cpu_offload?: boolean; + // Advanced (load-time) tuning. All optional; omit for the backend's auto defaults. + speed_mode?: "off" | "eager" | "default" | "max"; + transformer_quant?: "auto" | "int8" | "fp8" | "nvfp4" | "mxfp8"; + attention_backend?: + | "auto" + | "native" + | "cudnn" + | "flash" + | "flash2" + | "flash3" + | "flash4" + | "sage" + | "xformers" + | "aiter"; + memory_mode?: "auto" | "fast" | "balanced" | "low_vram"; + transformer_cache?: "off" | "fbcache"; } export interface DiffusionGenerateRequest { @@ -48,6 +76,16 @@ export interface DiffusionGenerateRequest { guidance?: number; seed?: number; batch_size?: number; + // Image-conditioned workflows. init_image alone = img2img; init_image + mask_image = + // inpaint. Base64 or data-URL. strength is the denoise amount (0 keeps source, 1 redraws). + init_image?: string; + mask_image?: string; + strength?: number; + // Upscale (hires fix): factor > 1 with an init_image enlarges the source and re-denoises + // it at low strength. Requires init_image; ignored for txt2img/inpaint/edit. + upscale?: number; + // Additional reference images for the FLUX.2 reference workflow, combined with init_image. + reference_images?: string[]; } // A persisted image's full generation recipe (also embedded in the PNG). diff --git a/studio/frontend/src/features/images/images-page.tsx b/studio/frontend/src/features/images/images-page.tsx index c832e38e4f..34d7d23b1a 100644 --- a/studio/frontend/src/features/images/images-page.tsx +++ b/studio/frontend/src/features/images/images-page.tsx @@ -9,6 +9,8 @@ import { Download01Icon, ImageAdd02Icon, InformationCircleIcon, + LayoutAlignRightIcon, + Settings02Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; @@ -28,6 +30,7 @@ import { } from "@/components/ui/select"; import { Slider } from "@/components/ui/slider"; import { Spinner } from "@/components/ui/spinner"; +import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; import { InfoHint } from "@/components/ui/info-hint"; import { ModelSelector } from "@/components/assistant-ui/model-selector"; @@ -67,6 +70,39 @@ const txt2img = (id: string, name: string): ModelOption => ({ description: "Text-to-image · GGUF", isGguf: true, }); + +// Instruction-editing GGUF (Qwen-Image-Edit). Same single-file GGUF flow as txt2img +// (the picker expands quant variants); the backend resolves it to the edit family, which +// exposes only the "edit" workflow. +const editGguf = (id: string, name: string): ModelOption => ({ + id, + name, + description: "Image editing · GGUF", + isGguf: true, +}); + +// How to load a curated non-GGUF (safetensors) model. "pipeline" = a full diffusers +// repo (from_pretrained, embedded bnb-4bit quant auto-applied); "single_file" = a +// single safetensors transformer (e.g. fp8) assembled onto its base repo. The backend +// gates these to unsloth/* repos. Keyed by repo id so the load handler knows the kind +// (and, for single_file, the exact filename). +type SafetensorsSpec = { kind: "pipeline" | "single_file"; filename?: string }; +const SAFETENSORS_MODELS: Record = { + "unsloth/Z-Image-Turbo-unsloth-bnb-4bit": { kind: "pipeline" }, + "unsloth/Qwen-Image-2512-unsloth-bnb-4bit": { kind: "pipeline" }, + "unsloth/Qwen-Image-2512-FP8": { + kind: "single_file", + filename: "qwen-image-2512-fp8.safetensors", + }, +}; +// Curated non-GGUF picker entries (isGguf:false -> no quant expander, direct load). +const safetensors = (id: string, name: string, label: string): ModelOption => ({ + id, + name, + description: `Text-to-image · ${label}`, + isGguf: false, +}); + const MODELS: ModelOption[] = [ txt2img("unsloth/Z-Image-Turbo-GGUF", "Z-Image-Turbo"), txt2img("unsloth/Z-Image-GGUF", "Z-Image"), @@ -76,6 +112,72 @@ const MODELS: ModelOption[] = [ txt2img("unsloth/FLUX.1-dev-GGUF", "FLUX.1 dev"), txt2img("unsloth/FLUX.2-klein-4B-GGUF", "FLUX.2 klein 4B"), txt2img("unsloth/FLUX.2-klein-9B-GGUF", "FLUX.2 klein 9B"), + editGguf("unsloth/Qwen-Image-Edit-2511-GGUF", "Qwen-Image-Edit 2511"), + editGguf("unsloth/FLUX.1-Kontext-dev-GGUF", "FLUX.1 Kontext dev"), + safetensors( + "unsloth/Z-Image-Turbo-unsloth-bnb-4bit", + "Z-Image-Turbo (bnb-4bit)", + "Safetensors · bnb-4bit", + ), + safetensors( + "unsloth/Qwen-Image-2512-unsloth-bnb-4bit", + "Qwen-Image 2512 (bnb-4bit)", + "Safetensors · bnb-4bit", + ), + safetensors( + "unsloth/Qwen-Image-2512-FP8", + "Qwen-Image 2512 (FP8)", + "Safetensors · fp8", + ), +]; + +// Workflow tabs. `requires` is the backend workflow id (status.workflows) that must +// be supported by the loaded model for the tab to enable; null = always available. +type WorkflowId = "create" | "transform" | "inpaint" | "extend" | "upscale" | "reference" | "edit"; + +const WORKFLOW_TABS: Array<{ + id: WorkflowId; + label: string; + requires: string | null; + hint?: string; +}> = [ + { id: "create", label: "Create", requires: null, hint: "Generate a new image from a prompt" }, + { + id: "transform", + label: "Transform", + requires: "img2img", + hint: "Redraw an uploaded image guided by your prompt (img2img)", + }, + { + id: "inpaint", + label: "Inpaint", + requires: "inpaint", + hint: "Paint over a region to regenerate just that area, keeping the rest", + }, + { + id: "extend", + label: "Extend", + requires: "outpaint", + hint: "Outpaint: grow the canvas and fill the new edges from your prompt", + }, + { + id: "upscale", + label: "Upscale", + requires: "upscale", + hint: "Hires fix: enlarge an uploaded image and re-detail it at higher resolution", + }, + { + id: "reference", + label: "Reference", + requires: "reference", + hint: "Generate a new image guided by a reference image + your prompt (FLUX.2)", + }, + { + id: "edit", + label: "Edit", + requires: "edit", + hint: "Instruction editing: change an image with a prompt (Qwen-Image-Edit)", + }, ]; // Per-model generation defaults (steps + guidance), matched by repo-id substring, @@ -88,6 +190,8 @@ const DEFAULT_GEN = { steps: 9, guidance: 0 }; const MODEL_DEFAULTS: Array<{ match: string; steps: number; guidance: number }> = [ { match: "z-image-turbo", steps: 9, guidance: 0 }, { match: "flux.1-schnell", steps: 4, guidance: 0 }, + // Kontext (editing) before the generic flux.1: ~28 steps, lower guidance (~2.5). + { match: "kontext", steps: 28, guidance: 2.5 }, { match: "flux.1", steps: 28, guidance: 3.5 }, { match: "flux.2-klein", steps: 4, guidance: 0 }, { match: "qwen-image", steps: 20, guidance: 4 }, @@ -284,7 +388,11 @@ function SliderField({ min={min} max={max} step={step} - className="w-12 text-right font-mono text-xs font-medium bg-muted/50 border border-border rounded-lg px-1.5 py-0.5 focus:outline-none focus:ring-1 focus:ring-primary/30 [&::-webkit-inner-spin-button]:appearance-none" + // The slider is the primary control, so the native number spinners are + // redundant — and on this narrow field their up/down arrows overlapped and + // covered the value. Remove them on every engine: appearance:textfield for + // Firefox, and zero out the webkit inner/outer spin buttons. + className="w-14 text-right font-mono text-xs font-medium bg-muted/50 border border-border rounded-lg px-1.5 py-0.5 focus:outline-none focus:ring-1 focus:ring-primary/30 [appearance:textfield] [&::-webkit-outer-spin-button]:m-0 [&::-webkit-outer-spin-button]:appearance-none [&::-webkit-inner-spin-button]:m-0 [&::-webkit-inner-spin-button]:appearance-none" /> @@ -312,6 +420,335 @@ function Field({ ); } +// A compact labeled Select row for the Advanced Options panel. +function AdvancedSelect({ + label, + hint, + value, + onValueChange, + options, +}: { + label: string; + hint?: ReactNode; + value: string; + onValueChange: (v: string) => void; + options: Array<[string, string]>; +}) { + return ( +
+ + {label} + {hint && {hint}} + + +
+ ); +} + +// Source-image picker for the Transform (img2img) workflow: click or drag-drop an +// image, read it to a data URL the generate request sends as init_image. Shows a +// thumbnail preview with a Clear button once an image is set. +function ImageDropzone({ + value, + onChange, +}: { + value: string | null; + onChange: (dataUrl: string | null) => void; +}) { + const inputRef = useRef(null); + const [dragging, setDragging] = useState(false); + + const readFile = useCallback( + (file: File | undefined | null) => { + if (!file || !file.type.startsWith("image/")) { + if (file) toast.error("Please choose an image file"); + return; + } + const reader = new FileReader(); + reader.onload = () => onChange(typeof reader.result === "string" ? reader.result : null); + reader.onerror = () => toast.error("Could not read the image"); + reader.readAsDataURL(file); + }, + [onChange], + ); + + if (value) { + return ( +
+ Source + +
+ ); + } + + return ( + + ); +} + +// A brush-based mask editor for inpainting. Shows the source image with a paintable +// overlay and exports a grayscale PNG mask at the image's NATIVE resolution, following +// the diffusers inpaint convention (white = repaint, black = keep). Strokes are drawn to +// both a visible tinted overlay (feedback) and an offscreen mask canvas kept in lockstep, +// so the exported mask always matches what the user sees. `brushPct` sizes the brush as a +// fraction of the image's shorter side, so it stays consistent across resolutions. +function MaskCanvas({ + image, + brushPct, + resetKey, + onMaskChange, +}: { + image: string; + brushPct: number; + resetKey: number; + onMaskChange: (dataUrl: string | null) => void; +}) { + const dispRef = useRef(null); + const maskRef = useRef(null); + const dims = useRef<{ w: number; h: number }>({ w: 0, h: 0 }); + const drawing = useRef(false); + const last = useRef<{ x: number; y: number } | null>(null); + const [ready, setReady] = useState(false); + + // (Re)initialise both canvases whenever the image changes or Clear is pressed: + // size them to the image's native pixels and reset the mask to all-black (keep all). + useEffect(() => { + setReady(false); + const img = new Image(); + img.onload = () => { + const w = img.naturalWidth; + const h = img.naturalHeight; + dims.current = { w, h }; + const disp = dispRef.current; + const mask = maskRef.current ?? document.createElement("canvas"); + maskRef.current = mask; + if (!disp) return; + disp.width = w; + disp.height = h; + mask.width = w; + mask.height = h; + const mctx = mask.getContext("2d"); + const dctx = disp.getContext("2d"); + if (!mctx || !dctx) return; + mctx.fillStyle = "#000"; + mctx.fillRect(0, 0, w, h); + dctx.clearRect(0, 0, w, h); + setReady(true); + onMaskChange(null); + }; + img.src = image; + }, [image, resetKey, onMaskChange]); + + const radius = useCallback(() => { + const base = Math.min(dims.current.w, dims.current.h) || 1024; + return Math.max(2, (brushPct / 100) * base); + }, [brushPct]); + + const toNatural = (e: React.PointerEvent) => { + const disp = dispRef.current; + if (!disp) return { x: 0, y: 0 }; + const r = disp.getBoundingClientRect(); + return { + x: ((e.clientX - r.left) / r.width) * dims.current.w, + y: ((e.clientY - r.top) / r.height) * dims.current.h, + }; + }; + + const stroke = (from: { x: number; y: number } | null, to: { x: number; y: number }) => { + const disp = dispRef.current; + const mask = maskRef.current; + if (!disp || !mask) return; + const r = radius(); + const layers: Array<[CanvasRenderingContext2D | null, string]> = [ + [disp.getContext("2d"), "rgba(244,114,114,0.55)"], + [mask.getContext("2d"), "#ffffff"], + ]; + for (const [ctx, style] of layers) { + if (!ctx) continue; + ctx.strokeStyle = style; + ctx.fillStyle = style; + ctx.lineWidth = r * 2; + ctx.lineCap = "round"; + ctx.lineJoin = "round"; + ctx.beginPath(); + ctx.arc(to.x, to.y, r, 0, Math.PI * 2); + ctx.fill(); + if (from) { + ctx.beginPath(); + ctx.moveTo(from.x, from.y); + ctx.lineTo(to.x, to.y); + ctx.stroke(); + } + } + }; + + const onDown = (e: React.PointerEvent) => { + if (!ready) return; + drawing.current = true; + try { + e.currentTarget.setPointerCapture(e.pointerId); + } catch { + // setPointerCapture can throw for synthetic events; safe to ignore. + } + const p = toNatural(e); + last.current = p; + stroke(null, p); + }; + const onMove = (e: React.PointerEvent) => { + if (!drawing.current) return; + const p = toNatural(e); + stroke(last.current, p); + last.current = p; + }; + const onUp = () => { + if (!drawing.current) return; + drawing.current = false; + last.current = null; + const mask = maskRef.current; + if (mask) onMaskChange(mask.toDataURL("image/png")); + }; + + return ( +
+ Inpaint source + +
+ ); +} + +function loadImage(src: string): Promise { + return new Promise((resolve, reject) => { + const img = new Image(); + img.onload = () => resolve(img); + img.onerror = reject; + img.src = src; + }); +} + +// Which sides to grow when outpainting. +type ExtendSides = { left: boolean; right: boolean; top: boolean; bottom: boolean }; + +// Build the (image, mask) pair for outpaint by reusing the inpaint backend: grow the +// canvas by `pct` of each dimension on the selected sides, edge-bleed the original pixels +// into the new bands (so the VAE encodes plausible content), and mask the new bands white +// (= repaint) with a small overlap into the original on each grown side so the seam blends. +async function buildOutpaint( + src: string, + sides: ExtendSides, + pct: number, +): Promise<{ image: string; mask: string }> { + const img = await loadImage(src); + const w = img.naturalWidth; + const h = img.naturalHeight; + const px = Math.round((pct / 100) * w); + const py = Math.round((pct / 100) * h); + const l = sides.left ? px : 0; + const r = sides.right ? px : 0; + const t = sides.top ? py : 0; + const b = sides.bottom ? py : 0; + const nw = w + l + r; + const nh = h + t + b; + + const ic = document.createElement("canvas"); + ic.width = nw; + ic.height = nh; + const ictx = ic.getContext("2d"); + if (!ictx) throw new Error("Could not build the extended canvas"); + ictx.drawImage(img, l, t, w, h); // original, centred by the chosen offsets + // Edge-bleed: stretch the 1px border strips into each new band (and corners). + if (l) ictx.drawImage(img, 0, 0, 1, h, 0, t, l, h); + if (r) ictx.drawImage(img, w - 1, 0, 1, h, l + w, t, r, h); + if (t) ictx.drawImage(img, 0, 0, w, 1, l, 0, w, t); + if (b) ictx.drawImage(img, 0, h - 1, w, 1, l, t + h, w, b); + if (l && t) ictx.drawImage(img, 0, 0, 1, 1, 0, 0, l, t); + if (r && t) ictx.drawImage(img, w - 1, 0, 1, 1, l + w, 0, r, t); + if (l && b) ictx.drawImage(img, 0, h - 1, 1, 1, 0, t + h, l, b); + if (r && b) ictx.drawImage(img, w - 1, h - 1, 1, 1, l + w, t + h, r, b); + + const overlap = Math.round(Math.min(w, h) * 0.02); + const ol = l ? overlap : 0; + const or = r ? overlap : 0; + const ot = t ? overlap : 0; + const ob = b ? overlap : 0; + const mc = document.createElement("canvas"); + mc.width = nw; + mc.height = nh; + const mctx = mc.getContext("2d"); + if (!mctx) throw new Error("Could not build the extend mask"); + mctx.fillStyle = "#ffffff"; // repaint everything... + mctx.fillRect(0, 0, nw, nh); + mctx.fillStyle = "#000000"; // ...except the kept original (inset by the seam overlap). + mctx.fillRect(l + ol, t + ot, w - ol - or, h - ot - ob); + + return { image: ic.toDataURL("image/png"), mask: mc.toDataURL("image/png") }; +} + // One labeled row in the recipe popover. function RecipeRow({ label, @@ -405,6 +842,55 @@ export function ImagesPage() { // Batch size = images per forward pass (VRAM-heavy); count = sequential loops. const [batchSize, setBatchSize] = useState(1); const [count, setCount] = useState(1); + // Active workflow tab. "create" = text-to-image; "transform" = img2img; "inpaint" = + // mask-guided redraw. More tabs (edit/extend/control/enhance) slot in here. + const [workflow, setWorkflow] = useState("create"); + // Transform (img2img) / Inpaint inputs: the uploaded source image as a data URL, and + // the denoise strength (how far to redraw it: low = keep source, high = reimagine). + const [initImage, setInitImage] = useState(null); + const [strength, setStrength] = useState(0.6); + // Inpaint mask (grayscale PNG data URL, white = repaint), the brush size as a percent + // of the image's shorter side, and a key bumped to clear the painted mask. + const [maskImage, setMaskImage] = useState(null); + const [brushPct, setBrushPct] = useState(8); + const [maskResetKey, setMaskResetKey] = useState(0); + // Extend (outpaint): how far to grow each dimension and which sides to grow. Reuses the + // inpaint backend by building a padded image + border mask at generate time. + const [extendPct, setExtendPct] = useState(25); + const [extendSides, setExtendSides] = useState({ + left: true, + right: true, + top: true, + bottom: true, + }); + // Upscale (hires fix): the enlargement factor and the (low) denoise strength used to + // re-detail the enlarged image. The backend caps the factor and rounds the target size. + const [upscaleFactor, setUpscaleFactor] = useState(2); + const [upscaleStrength, setUpscaleStrength] = useState(0.35); + // Reference (FLUX.2): up to 3 ADDITIONAL reference images beyond the primary one, combined + // by the model (subject + style, character + scene). + const [referenceImages, setReferenceImages] = useState([]); + // Advanced options live in a right-docked panel (like Chat's settings panel). Closed by + // default; a single fixed toggle in the top bar opens/closes it (the icon never moves). + const [advancedOpen, setAdvancedOpen] = useState(false); + // Advanced (load-time) options. "auto"/"off"/"none" map to the backend defaults + // (sent through on load). They apply when a model loads; changing them while a model + // is loaded shows a "Reapply" button that reloads the same model with the new values. + const [speedMode, setSpeedMode] = useState<"auto" | "off" | "eager" | "default" | "max">("auto"); + const [transformerQuant, setTransformerQuant] = useState< + "none" | "auto" | "int8" | "fp8" | "nvfp4" | "mxfp8" + >("none"); + const [attentionBackend, setAttentionBackend] = useState<"auto" | "native" | "cudnn" | "flash3" | "sage">( + "auto", + ); + const [memoryMode, setMemoryMode] = useState<"auto" | "fast" | "balanced" | "low_vram">("auto"); + const [transformerCache, setTransformerCache] = useState<"off" | "fbcache">("off"); + const [cpuOffload, setCpuOffload] = useState(false); + // The last load descriptor, so "Reapply" can reload the same model with new advanced + // options without the user re-picking it from the dropdown. + const lastLoad = useRef<{ repoId: string; kind: "gguf" | "single_file" | "pipeline"; filename?: string } | null>( + null, + ); const [busy, setBusy] = useState(null); // {done, total} while a multi-run generation is in flight (for the button). @@ -620,7 +1106,13 @@ export function ImagesPage() { }, [dismissLoadToast]); const handleLoad = useCallback( - async (repoId: string, ggufFilename: string) => { + async ( + repoId: string, + opts: { + kind: "gguf" | "single_file" | "pipeline"; + filename?: string; + }, + ) => { // Cancel any prior poll loop so two can't run at once. if (pollTimer.current) clearTimeout(pollTimer.current); setBusy("loading"); @@ -628,14 +1120,26 @@ export function ImagesPage() { dismissLoadToast(); lastLoadSig.current = null; loadToastId.current = toast(null, loadToastArgs(IDLE_PROGRESS)); + // Remember what was loaded so "Reapply" can reload it with new advanced options. + lastLoad.current = { repoId, kind: opts.kind, filename: opts.filename }; try { // Returns immediately — the load runs in the background; we poll for it. // The backend infers the family + base diffusers repo from the repo id. // Forward the saved HF token so gated bases (FLUX dev/klein) can download. + // A pipeline load carries no filename (the repo IS the pipeline); the + // single-file kinds send the GGUF / safetensors filename. Advanced options map + // sentinels ("auto"/"off"/"none") to omitted so the backend uses its defaults. await loadDiffusionModel({ model_path: repoId, - gguf_filename: ggufFilename, + model_kind: opts.kind, + gguf_filename: opts.filename, hf_token: hfApiToken(getHfToken()), + cpu_offload: cpuOffload, + speed_mode: speedMode === "auto" ? undefined : speedMode, + transformer_quant: transformerQuant === "none" ? undefined : transformerQuant, + attention_backend: attentionBackend === "auto" ? undefined : attentionBackend, + memory_mode: memoryMode === "auto" ? undefined : memoryMode, + transformer_cache: transformerCache === "off" ? undefined : transformerCache, }); } catch (err) { dismissLoadToast(); @@ -646,19 +1150,54 @@ export function ImagesPage() { } void pollLoadProgress(); }, - [pollLoadProgress, refreshStatus, dismissLoadToast], + [ + pollLoadProgress, + refreshStatus, + dismissLoadToast, + cpuOffload, + speedMode, + transformerQuant, + attentionBackend, + memoryMode, + transformerCache, + ], ); - // The chat picker emits (modelId, picked quant + its exact filename); load it, - // and seed the inputs with that model's defaults. + // Set (or clear) the Transform/Inpaint source image; always drop any painted mask so it + // can't be applied to a different image (the mask is sized to the previous source). + const handleInitChange = useCallback((dataUrl: string | null) => { + setInitImage(dataUrl); + setMaskImage(null); + setMaskResetKey((k) => k + 1); + }, []); + + // Reload the current model with the current advanced options. + const handleReapply = useCallback(() => { + const l = lastLoad.current; + if (l) void handleLoad(l.repoId, { kind: l.kind, filename: l.filename }); + }, [handleLoad]); + + // The chat picker emits (modelId, picked quant + its exact filename) for a GGUF, + // or just (modelId) for a curated non-GGUF safetensors pick; load it, and seed the + // inputs with that model's defaults. const handleModelSelect = useCallback( (id: string, meta: ModelSelectorChangeMeta) => { + // Curated non-GGUF model: load as a full pipeline or single-file safetensors. + const spec = SAFETENSORS_MODELS[id]; + if (spec) { + setQuant(null); + const d = defaultsFor(id); + setSteps(d.steps); + setGuidance(d.guidance); + void handleLoad(id, { kind: spec.kind, filename: spec.filename }); + return; + } if (!meta.ggufVariant || !meta.ggufFilename) return; // not a quant pick setQuant(meta.ggufVariant); const d = defaultsFor(id); setSteps(d.steps); setGuidance(d.guidance); - void handleLoad(id, meta.ggufFilename); + void handleLoad(id, { kind: "gguf", filename: meta.ggufFilename }); }, [handleLoad], ); @@ -681,6 +1220,80 @@ export function ImagesPage() { toast.error("Prompt is empty"); return; } + const isTransform = workflow === "transform"; + const isInpaint = workflow === "inpaint"; + const isExtend = workflow === "extend"; + const isUpscale = workflow === "upscale"; + const isReference = workflow === "reference"; + const isEdit = workflow === "edit"; + const usesInit = isTransform || isInpaint || isExtend || isUpscale || isReference || isEdit; + const tabLabel = isInpaint + ? "Inpaint" + : isExtend + ? "Extend" + : isUpscale + ? "Upscale" + : isReference + ? "Reference" + : isEdit + ? "Edit" + : "Transform"; + if (usesInit && !initImage) { + toast.error(`Upload a source image for ${tabLabel}`); + return; + } + if (isInpaint && !maskImage) { + toast.error("Paint a mask over the region to regenerate"); + return; + } + if (isExtend && !(extendSides.left || extendSides.right || extendSides.top || extendSides.bottom)) { + toast.error("Pick at least one side to extend"); + return; + } + + // Resolve the conditioning image/mask/strength for this workflow up front. Extend + // (outpaint) is built here from the source by padding + masking the new border, then + // sent through the same inpaint path. txt2img leaves all three undefined. + let condInit: string | undefined; + let condMask: string | undefined; + let condStrength: number | undefined; + let condUpscale: number | undefined; + let condRefImages: string[] | undefined; + try { + if (isTransform) { + condInit = initImage ?? undefined; + condStrength = strength; + } else if (isInpaint) { + condInit = initImage ?? undefined; + condMask = maskImage ?? undefined; + condStrength = strength; + } else if (isExtend) { + const built = await buildOutpaint(initImage!, extendSides, extendPct); + condInit = built.image; + condMask = built.mask; + condStrength = 1; // the new border is blank canvas: redraw it fully + } else if (isUpscale) { + // Hires fix: the backend enlarges the source by `upscale` and re-denoises it at + // this low strength so it gains detail without changing the content. + condInit = initImage ?? undefined; + condUpscale = upscaleFactor; + condStrength = upscaleStrength; + } else if (isReference) { + // FLUX.2 reference conditioning: send the primary reference + any extra references + // (combined by the model). The model generates a fresh image at the slider size + // guided by the references + prompt. No mask, no strength (not a denoise blend). + condInit = initImage ?? undefined; + const extras = referenceImages.filter(Boolean); + if (extras.length) condRefImages = extras; + } else if (isEdit) { + // Instruction editing: send the source image; the prompt IS the instruction. + // No mask, no strength (the edit pipeline fully regenerates from the instruction). + condInit = initImage ?? undefined; + } + } catch { + toast.error("Could not prepare the source image"); + return; + } // Resolve a base seed up front. With an explicit seed the run is fully // reproducible; with a random one we still pick a concrete base now so each // sequential image gets a distinct, reproducible seed (base + i). @@ -731,6 +1344,14 @@ export function ImagesPage() { guidance, seed: baseSeed + i, batch_size: batchSize, + // Transform/Inpaint/Extend send the source image (+ mask for inpaint/extend) and + // a denoise strength, resolved above. The backend derives output size from the + // image, so width/height are advisory here. + init_image: condInit, + mask_image: condMask, + strength: condStrength, + upscale: condUpscale, + reference_images: condRefImages, }); // Prepend this run's records (newest first) and load their blobs. setImages((prev) => [...res.images, ...prev]); @@ -747,14 +1368,128 @@ export function ImagesPage() { setGenDone(null); setGenStep(null); } - }, [prompt, negativePrompt, width, height, steps, guidance, seed, batchSize, count, ensureSrc]); + }, [prompt, negativePrompt, width, height, steps, guidance, seed, batchSize, count, workflow, initImage, maskImage, strength, extendPct, extendSides, upscaleFactor, upscaleStrength, referenceImages, ensureSrc]); + + // Keep the active workflow valid for the loaded model: an edit-only model (Qwen-Image- + // Edit) has no Create/Transform tabs, a base model has no Edit tab. Snap to the first + // supported workflow whenever the loaded model's capabilities change. + useEffect(() => { + if (!status?.loaded) return; + const wf = status.workflows ?? []; + const ok = (id: WorkflowId) => { + const t = WORKFLOW_TABS.find((x) => x.id === id); + if (!t) return false; + return t.requires === null ? wf.includes("txt2img") : wf.includes(t.requires); + }; + if (!ok(workflow)) { + const first = WORKFLOW_TABS.find((t) => ok(t.id)); + if (first) setWorkflow(first.id); + } + }, [status?.loaded, status?.workflows, workflow]); + + // The Advanced (load-time) tuning controls, rendered in the right-docked panel below. + const advancedControls = ( + <> + setSpeedMode(v as typeof speedMode)} + options={[ + ["auto", "Auto"], + ["off", "Off (bit-exact)"], + ["eager", "Eager"], + ["default", "Default (compile)"], + ["max", "Max"], + ]} + /> + {/* The dense transformer_quant fast path only engages on the GGUF kind; on a loaded + safetensors pipeline / single-file model it is a silent no-op, so gate the control + to GGUF (or nothing loaded) and otherwise show why it is unavailable. */} + {!status?.loaded || status.model_kind === "gguf" ? ( + setTransformerQuant(v as typeof transformerQuant)} + options={[ + ["none", "GGUF default"], + ["auto", "Auto (best for GPU)"], + ["fp8", "FP8"], + ["int8", "INT8"], + ["nvfp4", "NVFP4 (Blackwell)"], + ["mxfp8", "MXFP8 (Blackwell)"], + ]} + /> + ) : ( +
+ Transformer quant + GGUF models only +
+ )} + setAttentionBackend(v as typeof attentionBackend)} + options={[ + ["auto", "Auto"], + ["native", "Native SDPA"], + ["cudnn", "cuDNN"], + ["flash3", "FlashAttention 3"], + ["sage", "SageAttention (INT8)"], + ]} + /> + setMemoryMode(v as typeof memoryMode)} + options={[ + ["auto", "Auto"], + ["fast", "Fast (resident)"], + ["balanced", "Balanced"], + ["low_vram", "Low VRAM"], + ]} + /> + setTransformerCache(v as typeof transformerCache)} + options={[ + ["off", "Off"], + ["fbcache", "First-Block-Cache"], + ]} + /> +
+ + CPU offload + Offload to CPU to fit low-VRAM cards (slower). Overridden by Memory mode when that is not Auto. + + +
+ {status?.loaded && ( + + )} + + ); return (
{/* ── Top: the model selector, kept at the chat tab's exact position so the shared element matches. The load progress shows in a chat-style toast, not here. ── */} -
+
+ {/* Single fixed toggle for the right-docked Advanced panel (mirrors Chat's settings + toggle, same icon in both states so it never moves). Highlighted when open. */} +
{/* ── Controls rail + preview canvas. Padding mirrors the other tabs @@ -773,8 +1525,284 @@ export function ImagesPage() { {/* The controls rail. Plain card (the gray surface) with no header — the prompt + Generate button make the panel self-explanatory. */}
- -