Stop an ignored-cancel sd-server, guard deletes during diffusion training, repair unusable managed binaries
sd-server does not interrupt an in-flight job, so when it ignores a cancel the grace branch abandoned the poll and reported cancellation while the native job kept a core (or the GPU) busy to completion and held the server's job slot. The comment said the caller stops the server, but only unload does that immediately: a superseding load stops it after its multi-gigabyte download, and a load that then fails never gets there. Stop it here, as the deadline branch already does. DELETE /api/models/delete-finetuned checked only the LLM trainer, so it could rmtree the output directory a live diffusion LoRA run was about to write its adapter into. Consult the diffusion training service too, like the dataset mutation and model-load routes. find_sd_*_binary only checks is_file(), so an interrupted extraction (or a prebuilt for the wrong CPU) left a present-but-unrunnable binary the installer never retried: every load probed it, fell back to diffusers, and native inference stayed off until the directory was deleted by hand. Probe it and reinstall, but only for a copy under the installer-owned root -- SD_CLI_PATH, UNSLOTH_SD_CPP_PATH, an in-tree build and anything on PATH are the user's.
This commit is contained in:
parent
7dbc95936e
commit
a2342f80df
7 changed files with 162 additions and 14 deletions
|
|
@ -653,3 +653,54 @@ def test_print_asset_uses_upstream_fallback(monkeypatch, capsys):
|
|||
out = capsys.readouterr().out
|
||||
assert rc == 0
|
||||
assert "vulkan" in out and "no matching prebuilt" not in out
|
||||
|
||||
|
||||
def test_unrunnable_managed_binary_is_removed_so_it_reinstalls(monkeypatch, tmp_path):
|
||||
# An interrupted extraction leaves an sd-cli that exists but cannot run. The finder only checks
|
||||
# is_file(), so without a probe the installer never retried and native inference stayed off for
|
||||
# the life of the install.
|
||||
import core.inference.sd_cpp_backend as bk
|
||||
import core.inference.sd_cpp_engine as eng
|
||||
|
||||
root = tmp_path / "sd-home" / "stable-diffusion.cpp"
|
||||
root.mkdir(parents = True)
|
||||
managed = root / "sd-cli"
|
||||
managed.write_bytes(b"truncated")
|
||||
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "sd-home" / "studio"))
|
||||
assert eng.is_managed_binary(str(managed)) is True
|
||||
|
||||
monkeypatch.setattr(bk, "find_sd_cpp_binary", lambda: str(managed) if managed.exists() else None)
|
||||
monkeypatch.setattr(bk, "_server_binary_runnable", lambda *_a, **_k: False)
|
||||
installs: list = []
|
||||
|
||||
def _install(**kwargs):
|
||||
installs.append(kwargs)
|
||||
managed.write_bytes(b"good")
|
||||
return managed
|
||||
|
||||
import sys
|
||||
import types
|
||||
|
||||
stub = types.ModuleType("install_sd_cpp_prebuilt")
|
||||
stub.install = _install
|
||||
monkeypatch.setitem(sys.modules, "install_sd_cpp_prebuilt", stub)
|
||||
|
||||
out = bk.ensure_sd_cpp_binary(accelerator = "cpu")
|
||||
assert installs, "the unusable managed copy must trigger a reinstall"
|
||||
assert out == str(managed)
|
||||
|
||||
|
||||
def test_an_unrunnable_user_supplied_binary_is_never_deleted(monkeypatch, tmp_path):
|
||||
# SD_CLI_PATH / PATH / an in-tree build belong to the user: report them and let the router's own
|
||||
# probe refuse, but never remove or reinstall over them.
|
||||
import core.inference.sd_cpp_backend as bk
|
||||
|
||||
outside = tmp_path / "mine" / "sd-cli"
|
||||
outside.parent.mkdir(parents = True)
|
||||
outside.write_bytes(b"truncated")
|
||||
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path / "elsewhere" / "studio"))
|
||||
monkeypatch.setattr(bk, "find_sd_cpp_binary", lambda: str(outside))
|
||||
monkeypatch.setattr(bk, "_server_binary_runnable", lambda *_a, **_k: False)
|
||||
|
||||
assert bk.ensure_sd_cpp_binary(accelerator = "cpu") == str(outside)
|
||||
assert outside.exists(), "a user-supplied binary must survive"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue