Fix safetensors chat backend unload for PR #5754

_release_chat_backend_for_diffusion was importing
get_inference_backend from core.inference.inference (the in-subprocess
class) and calling unload_model() without the required model_name
argument. The TypeError was swallowed and the active chat model
stayed resident, defeating the chat-to-diffusion lifecycle handoff.

Switch to the orchestrator's accessor at core.inference and pass
active_model_name through, mirroring the GGUF chat-load path. Add a
regression test that stubs both backends and verifies unload_model
is called with the active model name.
This commit is contained in:
Daniel Han-Chen 2026-05-25 00:20:32 +00:00
commit 1601b7828f
2 changed files with 49 additions and 6 deletions

View file

@ -623,17 +623,23 @@ def _release_chat_backend_for_diffusion() -> None:
except Exception as exc:
logger.debug("llama-server unload skipped: %s", exc)
# 2. Safetensors / HF chat backend (the Inference orchestrator that
# 2. Safetensors / HF chat backend (the InferenceOrchestrator that
# serves FastVisionModel / FastLanguageModel weights). When this
# backend has a model resident on the same GPU, a diffusion load
# will OOM the same way.
# will OOM the same way. The orchestrator's unload_model takes a
# model_name; passing it without args raised TypeError and was
# swallowed, leaving the chat model resident.
try:
from core.inference.inference import get_inference_backend # type: ignore
from core.inference import get_inference_backend # type: ignore
backend = get_inference_backend()
if getattr(backend, "active_model_name", None):
logger.info("Unloading safetensors chat backend before diffusion load")
backend.unload_model()
active_model_name = getattr(backend, "active_model_name", None)
if active_model_name:
logger.info(
"Unloading safetensors chat backend '%s' before diffusion load",
active_model_name,
)
backend.unload_model(active_model_name)
except Exception as exc:
logger.debug("safetensors unload skipped: %s", exc)

View file

@ -477,6 +477,43 @@ def test_smart_base_repo_picks_base_4b(monkeypatch):
assert status["base_repo"] == "black-forest-labs/FLUX.2-klein-base-4B"
def test_release_chat_backend_calls_unload_with_model_name(monkeypatch):
"""The safetensors backend unload helper must call unload_model
with the active model name (the orchestrator's signature requires
it). The previous behaviour swallowed TypeError and left the chat
model resident, defeating the lifecycle handoff."""
import sys
import types
fake_pkg = types.ModuleType("core.inference")
calls: list = []
class _Stub:
active_model_name = "owner/some-model"
def unload_model(self, name):
calls.append(name)
self.active_model_name = None
return True
stub = _Stub()
fake_pkg.get_inference_backend = lambda: stub
monkeypatch.setitem(sys.modules, "core.inference", fake_pkg)
# Skip the llama-server branch by also stubbing routes.inference.
fake_routes = types.ModuleType("routes.inference")
fake_routes.get_llama_cpp_backend = lambda: types.SimpleNamespace(
is_loaded = False
)
monkeypatch.setitem(sys.modules, "routes.inference", fake_routes)
from core.inference.diffusion import _release_chat_backend_for_diffusion
_release_chat_backend_for_diffusion()
assert calls == ["owner/some-model"], calls
assert stub.active_model_name is None
def test_load_model_uses_safetensors_flag(monkeypatch):
"""The pipeline.from_pretrained call must pass use_safetensors=True
so pickle-backed .bin weights are refused at load time."""