Stop unverifiable records instead of skipping them, and record every bind address
This commit is contained in:
parent
bdd0cf3835
commit
07f2f468a1
4 changed files with 60 additions and 54 deletions
|
|
@ -689,26 +689,31 @@ def _get_pid_on_port(port: int) -> "tuple[int, str] | None":
|
|||
return None
|
||||
|
||||
|
||||
def _bind_address(host: str, port: int) -> str:
|
||||
"""The address a bind to *host* actually uses -- resolved exactly as
|
||||
_is_port_free does, so a recorded address and a requested one normalize alike."""
|
||||
def _bind_addresses(host: str, port: int) -> "set[str]":
|
||||
"""Every address *host* resolves to. `localhost` is both 127.0.0.1 and ::1, and
|
||||
recording only the first lets a later launch on the other one miss us."""
|
||||
import socket
|
||||
|
||||
try:
|
||||
return socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)[0][4][0]
|
||||
except (OSError, IndexError):
|
||||
return host
|
||||
infos = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
|
||||
except OSError:
|
||||
return {host}
|
||||
return {info[4][0] for info in infos} or {host}
|
||||
|
||||
|
||||
def _addresses_collide(recorded: "str | None", host: str, port: int) -> bool:
|
||||
"""Would a server bound to *recorded* block a bind to *host*?
|
||||
|
||||
Unknown or wildcard on either side collides: refusing with a clear message
|
||||
beats silently starting a duplicate.
|
||||
*recorded* may list several addresses. Unknown or wildcard on either side
|
||||
collides: refusing with a clear message beats silently starting a duplicate.
|
||||
"""
|
||||
wildcards = ("0.0.0.0", "::", "")
|
||||
if not recorded or recorded in wildcards or host in wildcards:
|
||||
if not recorded or host in wildcards:
|
||||
return True
|
||||
return recorded == _bind_address(host, port)
|
||||
listed = {a.strip() for a in recorded.split(",") if a.strip()}
|
||||
if not listed or listed & set(wildcards):
|
||||
return True
|
||||
return bool(listed & _bind_addresses(host, port))
|
||||
|
||||
|
||||
def _is_port_free(host: str, port: int) -> bool:
|
||||
|
|
@ -961,7 +966,7 @@ def _write_pid_file(port: int, host: str = ""):
|
|||
# Start time pins the record to this process; the bind address tells a
|
||||
# later launch whether this server would actually block it.
|
||||
created = _process_create_time(os.getpid())
|
||||
address = _bind_address(host, port) if host else ""
|
||||
address = ",".join(sorted(_bind_addresses(host, port))) if host else ""
|
||||
body = f"{os.getpid()}\n{'' if created is None else repr(created)}\n{address}"
|
||||
path.write_text(body, encoding = "utf-8")
|
||||
# An older CLI's `stop` only reads this one, and expects a bare PID.
|
||||
|
|
|
|||
|
|
@ -322,11 +322,29 @@ def test_address_matching(tmp_path):
|
|||
def test_a_hostname_resolves_the_same_way_the_bind_does(tmp_path):
|
||||
# `localhost` and the address _is_port_free actually binds must agree, or a
|
||||
# recorded server is missed and a duplicate starts.
|
||||
recorded = run._bind_address("localhost", 8889)
|
||||
recorded = ",".join(sorted(run._bind_addresses("localhost", 8889)))
|
||||
|
||||
assert run._addresses_collide(recorded, "localhost", 8889) is True
|
||||
|
||||
|
||||
def test_a_hostname_records_every_address_it_resolves_to(tmp_path):
|
||||
# `localhost` binds 127.0.0.1 AND ::1. Recording only the first lets a later
|
||||
# launch on the other literal miss us and start a duplicate.
|
||||
addrs = run._bind_addresses("localhost", 8889)
|
||||
recorded = ",".join(sorted(addrs))
|
||||
|
||||
for literal in addrs:
|
||||
assert run._addresses_collide(recorded, literal, 8889) is True
|
||||
|
||||
|
||||
def test_a_multi_address_record_matches_either_literal(tmp_path):
|
||||
recorded = "127.0.0.1,::1"
|
||||
|
||||
assert run._addresses_collide(recorded, "127.0.0.1", 8889) is True
|
||||
assert run._addresses_collide(recorded, "::1", 8889) is True
|
||||
assert run._addresses_collide("127.0.0.1", "::1", 8889) is False
|
||||
|
||||
|
||||
def test_fallback_aborts_on_our_own_server_further_up_the_range(tmp_path, monkeypatch):
|
||||
# jupyter holds 8888, our server holds 8889: skipping to 8890 is the duplicate.
|
||||
(tmp_path / "studio-8889-8550.pid").write_text("8550\n\n127.0.0.1", encoding = "utf-8")
|
||||
|
|
|
|||
|
|
@ -2475,12 +2475,12 @@ def _pid_file_entries() -> "list[tuple[int, list[float | None], list[Path]]]":
|
|||
return [(pid, times, files) for pid, (times, files) in by_pid.items()]
|
||||
|
||||
|
||||
def _pid_is_studio_server(pid: int, created_times: "Sequence[float | None]" = ()) -> "bool | None":
|
||||
"""Guard against PID reuse. True = ours, False = not ours, None = can't tell.
|
||||
def _pid_is_studio_server(pid: int, created_times: "Sequence[float | None]" = ()) -> bool:
|
||||
"""Best-effort PID-reuse guard: False only when we can prove it isn't ours.
|
||||
|
||||
psutil is not a base CLI dependency but the managed backend has it, so the CLI
|
||||
can meet timestamped records it cannot verify. Those are None, never False:
|
||||
deleting one silently orphans a live server, which is the bug this fixes.
|
||||
psutil is not a base CLI dependency, so the CLI meets records it cannot check.
|
||||
Those are stopped anyway -- the old `stop` signalled its PID with no checks at
|
||||
all, and skipping a live server is the orphan bug this exists to fix.
|
||||
The cmdline check covers untimed legacy records -- the in-venv path runs
|
||||
run_server() in-process, so its argv is `unsloth studio ...` with no run.py.
|
||||
"""
|
||||
|
|
@ -2488,19 +2488,14 @@ def _pid_is_studio_server(pid: int, created_times: "Sequence[float | None]" = ()
|
|||
untimed = not created_times or len(known) < len(created_times)
|
||||
try:
|
||||
import psutil
|
||||
|
||||
proc = psutil.Process(pid)
|
||||
except Exception:
|
||||
return True if untimed else None
|
||||
if known:
|
||||
try:
|
||||
if known:
|
||||
actual = proc.create_time()
|
||||
except Exception:
|
||||
return True if untimed else None
|
||||
if any(abs(actual - c) < 1.0 for c in known):
|
||||
return True
|
||||
if not untimed:
|
||||
return False
|
||||
try:
|
||||
if any(abs(actual - c) < 1.0 for c in known):
|
||||
return True
|
||||
if not untimed:
|
||||
return False
|
||||
cmdline = " ".join(proc.cmdline()).lower()
|
||||
except Exception:
|
||||
return True
|
||||
|
|
@ -2537,19 +2532,9 @@ def stop():
|
|||
typer.echo("No running Unsloth server found (no PID file).")
|
||||
raise typer.Exit(0)
|
||||
|
||||
signalled, failed, unverified = [], [], []
|
||||
signalled, failed = [], []
|
||||
for pid, created_times, paths in entries:
|
||||
identity = _pid_is_studio_server(pid, created_times) if _pid_alive(pid) else False
|
||||
if identity is None:
|
||||
# Keep the record: deleting it is what strands a live server.
|
||||
unverified.append(pid)
|
||||
typer.echo(
|
||||
f"Cannot confirm PID {pid} is an Unsloth server (install psutil to check); "
|
||||
"leaving it alone. Stop it manually if it is one.",
|
||||
err = True,
|
||||
)
|
||||
continue
|
||||
if not identity:
|
||||
if not _pid_alive(pid) or not _pid_is_studio_server(pid, created_times):
|
||||
for path in paths:
|
||||
path.unlink(missing_ok = True)
|
||||
continue
|
||||
|
|
@ -2562,8 +2547,6 @@ def stop():
|
|||
signalled.append((pid, paths))
|
||||
|
||||
if not signalled and not failed:
|
||||
if unverified:
|
||||
raise typer.Exit(1)
|
||||
typer.echo("No running Unsloth server found (cleaned up stale PID files).")
|
||||
raise typer.Exit(0)
|
||||
|
||||
|
|
@ -2584,7 +2567,7 @@ def stop():
|
|||
typer.echo(f"Unsloth server{'s' if stopped > 1 else ''} stopped ({stopped}).")
|
||||
for pid, _paths in pending:
|
||||
typer.echo(f"Unsloth server (PID {pid}) is shutting down (may take a few seconds).")
|
||||
if failed or unverified:
|
||||
if failed:
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -184,18 +184,20 @@ def test_pid_identity_check_accepts_an_in_process_studio(monkeypatch):
|
|||
assert studio_mod._pid_is_studio_server(8550) is True
|
||||
|
||||
|
||||
def test_a_timestamped_record_is_unverifiable_without_psutil(monkeypatch):
|
||||
# psutil is not a base CLI dependency but the managed backend has it, so the
|
||||
# CLI meets records it cannot check. Unknown, not "not ours".
|
||||
def test_an_unverifiable_record_is_still_stopped(monkeypatch):
|
||||
# psutil is not a base CLI dependency, so the CLI meets timestamped records it
|
||||
# cannot check. The old `stop` signalled with no checks at all -- skipping one
|
||||
# would leave a live server running, the orphan bug this exists to fix.
|
||||
studio_mod = _studio()
|
||||
monkeypatch.setitem(sys.modules, "psutil", None)
|
||||
|
||||
assert studio_mod._pid_is_studio_server(8550, [111.5]) is None
|
||||
assert studio_mod._pid_is_studio_server(8550, [111.5]) is True
|
||||
assert studio_mod._pid_is_studio_server(8550, [None]) is True
|
||||
|
||||
|
||||
def test_stop_keeps_an_unverifiable_record_instead_of_deleting_it(monkeypatch, tmp_path):
|
||||
# Deleting it strands a live server with no record -- the bug this PR fixes.
|
||||
def test_stop_signals_a_timestamped_record_without_psutil(monkeypatch, tmp_path):
|
||||
# Multiple servers on different ports: only the newest is also in studio.pid,
|
||||
# so the earlier ones are timestamp-only and must still be stopped.
|
||||
studio_mod, _live, killed = _install(monkeypatch, tmp_path, alive = {8550})
|
||||
monkeypatch.setattr(studio_mod, "_pid_is_studio_server", _REAL_IS_STUDIO_SERVER)
|
||||
monkeypatch.setitem(sys.modules, "psutil", None)
|
||||
|
|
@ -203,11 +205,9 @@ def test_stop_keeps_an_unverifiable_record_instead_of_deleting_it(monkeypatch, t
|
|||
|
||||
result = _run_stop(studio_mod)
|
||||
|
||||
combined = (result.output or "") + (getattr(result, "stderr", "") or "")
|
||||
assert result.exit_code == 1, combined
|
||||
assert killed == []
|
||||
assert (tmp_path / "studio-8901-8550.pid").exists()
|
||||
assert "cannot confirm pid 8550" in combined.lower()
|
||||
assert result.exit_code == 0, result.output
|
||||
assert killed == [8550]
|
||||
assert not (tmp_path / "studio-8901-8550.pid").exists()
|
||||
|
||||
|
||||
def test_a_legacy_record_survives_a_stale_timestamp_for_the_same_pid(monkeypatch):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue