Clarify model reuse and switching output
This commit is contained in:
parent
26b341bde6
commit
751fa0bfaf
2 changed files with 24 additions and 11 deletions
|
|
@ -634,8 +634,12 @@ def _load_model_with_progress(
|
|||
|
||||
threading.Thread(target = _load, name = "unsloth-model-load", daemon = True).start()
|
||||
progress = _ModelDownloadProgress(base, key, model, load.gguf_variant)
|
||||
loading_announced = False
|
||||
try:
|
||||
while not done.wait(_DOWNLOAD_POLL_INTERVAL_S):
|
||||
if not loading_announced:
|
||||
typer.echo(f"Loading model: {_display_model_spec(model, load.gguf_variant)}")
|
||||
loading_announced = True
|
||||
progress.poll()
|
||||
ok, value = result[0]
|
||||
if not ok:
|
||||
|
|
@ -1102,6 +1106,7 @@ def _resolve_model(
|
|||
load: LoadOptions = LoadOptions(),
|
||||
) -> dict:
|
||||
models = _loaded_models(base, key)
|
||||
load_requested = False
|
||||
# Only casefold-match ids against a loopback Unsloth, where _is_hub_model_id's
|
||||
# local existence probe can actually reject a server-side path; see the note there.
|
||||
allow_casefold = is_loopback_url(base)
|
||||
|
|
@ -1131,6 +1136,7 @@ def _resolve_model(
|
|||
)
|
||||
)
|
||||
if requested and match is None:
|
||||
load_requested = True
|
||||
active = next((m for m in models if m.get("loaded") is not False), None)
|
||||
active_id = active.get("id") if active else None
|
||||
if active_id and not _model_id_matches(
|
||||
|
|
@ -1138,11 +1144,8 @@ def _resolve_model(
|
|||
requested,
|
||||
allow_casefold = allow_casefold,
|
||||
):
|
||||
typer.echo(
|
||||
f"Switching the Unsloth server from {active_id} to {requested}. "
|
||||
"This unloads the current model for every attached session."
|
||||
)
|
||||
typer.echo(f"Loading model: {_display_model_spec(requested, load.gguf_variant)}")
|
||||
typer.echo(f"Switching the Unsloth server from {active_id} to {requested}.")
|
||||
typer.echo("This unloads the current model for every attached session.")
|
||||
# Mirror `unsloth run`'s load knobs; keep the default payload as just
|
||||
# model_path so a bare `--model` load is unchanged.
|
||||
payload = {"model_path": requested}
|
||||
|
|
@ -1155,6 +1158,8 @@ def _resolve_model(
|
|||
if load.tensor_parallel:
|
||||
payload["tensor_parallel"] = True
|
||||
loaded = _load_model_with_progress(base, key, requested, load, payload)
|
||||
if loaded.get("status") == "already_loaded":
|
||||
typer.echo(f"Reusing loaded model: {_display_model_spec(requested, load.gguf_variant)}")
|
||||
# Unsloth registers the model under a canonical id (resolved identifier,
|
||||
# casing) that /v1/models echoes but which may differ from the path we
|
||||
# passed; match on the id the load reports so we don't silently fall
|
||||
|
|
@ -1174,6 +1179,8 @@ def _resolve_model(
|
|||
None,
|
||||
)
|
||||
if match is not None:
|
||||
if requested and not load_requested:
|
||||
typer.echo(f"Reusing loaded model: {_display_model_spec(requested, load.gguf_variant)}")
|
||||
return match
|
||||
if requested:
|
||||
# We asked Unsloth to load it and it didn't surface in /v1/models; don't
|
||||
|
|
|
|||
|
|
@ -640,8 +640,13 @@ def fake_studio(tmp_path, monkeypatch):
|
|||
if url.endswith("/api/auth/api-keys"):
|
||||
return {"key": "sk-unsloth-feedfacefeedface"}
|
||||
if url.endswith("/api/inference/load"):
|
||||
already_loaded = state["models"][0]["id"] == payload["model_path"]
|
||||
state["models"] = [{"id": payload["model_path"], "context_length": 4096}]
|
||||
return {}
|
||||
return {
|
||||
"status": "already_loaded" if already_loaded else "loaded",
|
||||
"model": payload["model_path"],
|
||||
"display_name": payload["model_path"],
|
||||
}
|
||||
raise AssertionError(f"unexpected request: {method} {url}")
|
||||
|
||||
monkeypatch.setattr(start, "find_studio_server", lambda: BASE)
|
||||
|
|
@ -864,7 +869,6 @@ def test_resolve_model_matches_loaded_canonical_case_after_load(monkeypatch, cap
|
|||
assert entry["id"] == "unsloth/gemma-4-E2B-it-GGUF"
|
||||
assert any(c[1].endswith("/api/inference/load") for c in calls)
|
||||
output = capsys.readouterr().out
|
||||
assert "Loading model: unsloth/gemma-4-e2b-it-gguf:UD-Q4_K_XL\n" in output
|
||||
assert "please wait" not in output
|
||||
|
||||
|
||||
|
|
@ -1236,11 +1240,11 @@ def test_connect_model_flag_loads_on_server(fake_studio):
|
|||
assert loads == [
|
||||
("POST", f"{BASE}/api/inference/load", {"model_path": "unsloth/Qwen3.5-35B-A3B"})
|
||||
]
|
||||
assert (
|
||||
f"Switching the Unsloth server from {MODEL['id']} to unsloth/Qwen3.5-35B-A3B"
|
||||
in result.output
|
||||
assert result.output.index(
|
||||
f"Switching the Unsloth server from {MODEL['id']} to unsloth/Qwen3.5-35B-A3B.\n"
|
||||
) < result.output.index(
|
||||
"This unloads the current model for every attached session.\n"
|
||||
)
|
||||
assert "unloads the current model for every attached session" in result.output
|
||||
_assert_env_set(result.output, "ANTHROPIC_MODEL", "unsloth/Qwen3.5-35B-A3B")
|
||||
|
||||
|
||||
|
|
@ -1331,6 +1335,7 @@ def test_connect_model_bare_id_matches_loaded_without_reload(fake_studio):
|
|||
assert result.exit_code == 0, result.output
|
||||
loads = [c for c in fake_studio if c[1].endswith("/api/inference/load")]
|
||||
assert loads == []
|
||||
assert f"Reusing loaded model: {MODEL['id']}\n" in result.output
|
||||
_assert_env_set(result.output, "ANTHROPIC_MODEL", MODEL["id"])
|
||||
|
||||
|
||||
|
|
@ -1352,6 +1357,7 @@ def test_connect_model_variant_suffix_defers_to_server_dedup(fake_studio):
|
|||
{"model_path": MODEL["id"], "gguf_variant": "UD-Q4_K_XL"},
|
||||
)
|
||||
]
|
||||
assert f"Reusing loaded model: {MODEL['id']}:UD-Q4_K_XL\n" in result.output
|
||||
_assert_env_set(result.output, "ANTHROPIC_MODEL", MODEL["id"])
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue