diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index 510c3543d2..b353f0ec83 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -183,16 +183,16 @@ jobs: # available to llama.cpp from CI; gemma-3-270m turn latency # has been observed to crowd the 180s default. Triple it. STUDIO_UI_TURN_TIMEOUT_MS: '540000' - # Retry up to 3 times to absorb the racy Playwright Node 24 - # pipeTransport.js 'Unexpected end of JSON input' crash that - # fires intermittently on macos-14 free runners (Chromium - # browser process dies mid-test → driver Node process can't - # parse the truncated JSON-RPC line and exits). The retry - # FULLY resets Studio (kill, reset-password, reboot, wait - # /api/health, re-export bootstrap pw) before re-running the - # script so the change-password flow finds a fresh bootstrap. - # A real test failure (assertion / timeout) does NOT match the - # JSON pattern so it bypasses retry and surfaces immediately. + # Retry up to 3 times to absorb known macos-14 free-runner + # flakes: (1) Playwright Node 24 pipeTransport.js 'Unexpected + # end of JSON input' crash when the Chromium browser process + # dies mid-test, and (2) Chromium net::ERR_NO_BUFFER_SPACE + # when the runner's kernel briefly runs out of socket buffers. + # The retry FULLY resets Studio (kill, reset-password, reboot, + # wait /api/health, re-export bootstrap pw) before re-running + # the script. A real test failure (assertion / timeout) does + # NOT match either pattern so it bypasses retry and surfaces + # immediately. run: | mkdir -p logs/playwright attempt=1 @@ -205,9 +205,10 @@ jobs: if [ "$rc" -eq 0 ]; then break fi - if grep -q "Unexpected end of JSON input" logs/playwright_attempt_${attempt}.log \ + if { grep -q "Unexpected end of JSON input" logs/playwright_attempt_${attempt}.log \ + || grep -q "ERR_NO_BUFFER_SPACE" logs/playwright_attempt_${attempt}.log; } \ && [ "$attempt" -lt "$max_attempts" ]; then - echo "::warning::Playwright pipeTransport JSON crash on attempt ${attempt}; resetting Studio and retrying..." + echo "::warning::Playwright flake on attempt ${attempt}; resetting Studio and retrying..." kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 unsloth studio reset-password @@ -280,8 +281,8 @@ jobs: STUDIO_UI_TURN_TIMEOUT_MS: '540000' GGUF_REPO: ${{ env.GGUF_REPO }} GGUF_VARIANT: ${{ env.GGUF_VARIANT }} - # Same pipeTransport JSON-crash retry shape as "Drive the chat - # UI with Playwright" -- see comment there. + # Same flake-retry shape as "Drive the chat UI with Playwright" + # -- catches pipeTransport JSON crash and ERR_NO_BUFFER_SPACE. run: | mkdir -p logs/playwright_extra attempt=1 @@ -294,9 +295,10 @@ jobs: if [ "$rc" -eq 0 ]; then break fi - if grep -q "Unexpected end of JSON input" logs/playwright_extra_attempt_${attempt}.log \ + if { grep -q "Unexpected end of JSON input" logs/playwright_extra_attempt_${attempt}.log \ + || grep -q "ERR_NO_BUFFER_SPACE" logs/playwright_extra_attempt_${attempt}.log; } \ && [ "$attempt" -lt "$max_attempts" ]; then - echo "::warning::Playwright pipeTransport JSON crash on attempt ${attempt}; resetting Studio and retrying..." + echo "::warning::Playwright flake on attempt ${attempt}; resetting Studio and retrying..." kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true sleep 2 unsloth studio reset-password diff --git a/studio/backend/run.py b/studio/backend/run.py index 0787e04c47..d5ccc49022 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -24,7 +24,7 @@ if str(backend_dir) not in sys.path: import _platform_compat # noqa: F401 from loggers import get_logger -from startup_banner import print_studio_access_banner +from startup_banner import print_studio_access_banner, print_studio_stop_hint logger = get_logger(__name__) @@ -74,6 +74,255 @@ def _resolve_external_ip() -> str: return "0.0.0.0" +def _install_uvicorn_startup_log_rewrite(bind_host: str, display_host: str) -> None: + """Rewrite Uvicorn's startup log line: swap wildcard bind for the + externally-reachable address, replace the CTRL+C suffix with our Mac-aware + stop hint, and rename the prefix to "Unsloth Studio running on".""" + import logging + import re + + rewrite_host = ( + bind_host in ("0.0.0.0", "::") + and bool(display_host) + and display_host != bind_host + ) + new_suffix = "(To stop: press Ctrl+C -- on macOS, Control+C not Command+C)" + old_suffix_re = re.compile(r"\(Press CTRL\+C to quit\)") + old_prefix = "Uvicorn running on " + new_prefix = "Unsloth Studio running on " + + def _rewrite(text: str) -> str: + if text.startswith(old_prefix): + text = new_prefix + text[len(old_prefix) :] + return old_suffix_re.sub(new_suffix, text) + + class _UvicornStartupRewrite(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + try: + msg = record.msg if isinstance(record.msg, str) else "" + if ( + msg.startswith(old_prefix) + and isinstance(record.args, tuple) + and len(record.args) >= 3 + ): + if rewrite_host and record.args[1] == bind_host: + record.args = ( + record.args[0], + display_host, + record.args[2], + *record.args[3:], + ) + record.msg = _rewrite(msg) + cmsg = getattr(record, "color_message", None) + if isinstance(cmsg, str): + record.color_message = _rewrite(cmsg) + except Exception: + pass + return True + + f = _UvicornStartupRewrite() + for name in ("uvicorn", "uvicorn.error"): + logging.getLogger(name).addFilter(f) + + +def _local_port_open(host: str, port: int, timeout: float = 1.0) -> bool: + """Return True iff a TCP connection to (host, port) succeeds within timeout.""" + import socket + + try: + with socket.create_connection((host, port), timeout = timeout): + return True + except OSError: + return False + + +def _working_local_url(port: int) -> "str | None": + """Return a working loopback URL on this machine, or None if neither + 127.0.0.1 nor ::1 responds. Used as a fallback when external reachability fails.""" + if _local_port_open("127.0.0.1", port): + return f"http://127.0.0.1:{port}" + if _local_port_open("::1", port): + return f"http://[::1]:{port}" + return None + + +def _stdout_color_ok() -> bool: + """Whether to emit ANSI color codes on stdout. Mirrors startup_banner.""" + if os.environ.get("NO_COLOR", "").strip(): + return False + if os.environ.get("FORCE_COLOR", "").strip(): + return True + try: + return sys.stdout.isatty() + except (AttributeError, OSError, ValueError): + return False + + +def _verify_global_reachability(display_host: str, port: int) -> None: + """Probe check-host.net to confirm display_host:port is reachable from the + public internet. Synchronous so the caller can render output between the + banner URL section and the trailing stop hint. Bounded at ~15s; failures + are swallowed (the verifier failing is not Studio failing). Only meaningful + when bound to a wildcard host.""" + import ipaddress + import json + import time + import urllib.error + import urllib.parse + import urllib.request + + if not display_host or display_host in ("0.0.0.0", "::"): + return + + use_color = _stdout_color_ok() + dim = "\033[38;5;245m" if use_color else "" + ok_c = "\033[38;5;120;1m" if use_color else "" + err_c = "\033[38;5;203;1m" if use_color else "" + warn_c = "\033[38;5;215;1m" if use_color else "" + local_url_c = "\033[38;5;108;1m" if use_color else "" # matches banner's URL color + reset = "\033[0m" if use_color else "" + + url = f"http://{display_host}:{port}" + + # Private / loopback / link-local addresses are not globally routable. + try: + addr = ipaddress.ip_address(display_host) + if addr.is_loopback or addr.is_private or addr.is_link_local: + print( + f"{dim} Note: {display_host} is a private/LAN address -- " + f"reachable on this network only, not from the public internet." + f"{reset}", + flush = True, + ) + return + except ValueError: + # Not an IP literal; probe by hostname. + pass + + try: + qs = urllib.parse.urlencode({"host": f"{display_host}:{port}", "max_nodes": 3}) + req = urllib.request.Request( + f"https://check-host.net/check-tcp?{qs}", + headers = { + "Accept": "application/json", + "User-Agent": "unsloth-studio-reachability/1", + }, + ) + with urllib.request.urlopen(req, timeout = 5) as resp: + init = json.loads(resp.read().decode("utf-8", errors = "replace")) + req_id = init.get("request_id") + if not req_id: + return + + results = {} + deadline = time.monotonic() + 15.0 + poll_req = urllib.request.Request( + f"https://check-host.net/check-result/{req_id}", + headers = { + "Accept": "application/json", + "User-Agent": "unsloth-studio-reachability/1", + }, + ) + while time.monotonic() < deadline: + time.sleep(1.5) + try: + with urllib.request.urlopen(poll_req, timeout = 5) as resp: + results = json.loads(resp.read().decode("utf-8", errors = "replace")) + except Exception: + continue + if results and all(v is not None for v in results.values()): + break + # Two decisive nodes is enough; stop polling early. + decisive = [ + v + for v in results.values() + if isinstance(v, list) + and v + and isinstance(v[0], dict) + and ("time" in v[0] or "error" in v[0]) + ] + if len(decisive) >= 2: + break + + ok_nodes = err_nodes = 0 + for v in results.values(): + if not isinstance(v, list) or not v or not isinstance(v[0], dict): + continue + if "time" in v[0]: + ok_nodes += 1 + elif "error" in v[0]: + err_nodes += 1 + total = ok_nodes + err_nodes + + print("", flush = True) + if ok_nodes: + print( + f"{ok_c} Reachability check: {url}/ is reachable from the " + f"public internet ({ok_nodes}/{total} probe nodes connected).{reset}", + flush = True, + ) + elif err_nodes: + print( + f"{err_c} Reachability check: {url}/ is NOT reachable from " + f"the public internet ({err_nodes}/{total} probe nodes failed).{reset}", + flush = True, + ) + print(f"{dim} Common causes:{reset}", flush = True) + print( + f"{dim} * AWS -- the instance's Security Group doesn't " + f"allow inbound TCP {port}.{reset}", + flush = True, + ) + print( + f"{dim} * GCP -- no firewall rule allowing TCP {port} " + f"for the instance's network tag.{reset}", + flush = True, + ) + print( + f"{dim} * Azure / other clouds -- equivalent NSG / " + f"firewall rule missing.{reset}", + flush = True, + ) + print( + f"{dim} * Home -- your router isn't port-forwarding " + f"{port} to this machine.{reset}", + flush = True, + ) + print( + f"{dim} Workaround that needs no firewall changes -- " + f"SSH local-forward from your laptop:{reset}", + flush = True, + ) + print( + f"{dim} ssh -L {port}:localhost:{port} " + f"@{display_host}{reset}", + flush = True, + ) + print( + f"{dim} then open http://localhost:{port}/ in your browser.{reset}", + flush = True, + ) + # Only offer the local URL if loopback actually answers. + local_url = _working_local_url(port) + if local_url: + print( + f"{local_url_c} You can access Unsloth Studio locally " + f"in the meantime: {local_url}{reset}", + flush = True, + ) + else: + print( + f"{warn_c} Reachability check: probe nodes did not respond " + f"in time -- could not verify {url}/.{reset}", + flush = True, + ) + except urllib.error.URLError: + # Outbound HTTPS blocked; skip silently. + pass + except Exception: + pass + + def _get_pid_on_port(port: int) -> "tuple[int, str] | None": """Return (pid, process_name) of the process listening on *port*, or None. @@ -344,6 +593,10 @@ def run_server( if not silent: print(f"[WARNING] Frontend not found at {frontend_path}") + # Resolve once; shared by the log rewrite and the banner. + display_host = _resolve_external_ip() if host == "0.0.0.0" else host + _install_uvicorn_startup_log_rewrite(host, display_host) + ready_event = Event() startup_failed = Event() startup_errors = [] @@ -426,12 +679,18 @@ def run_server( print(f"TAURI_PORT={port}", flush = True) if not silent: - display_host = _resolve_external_ip() if host == "0.0.0.0" else host + wildcard_bind = host in ("0.0.0.0", "::") + # For wildcard binds, run the reachability check between the URL + # section and the stop hint so the stop hint stays last on screen. print_studio_access_banner( port = port, bind_host = host, display_host = display_host, + include_stop_hint = not wildcard_bind, ) + if wildcard_bind: + _verify_global_reachability(display_host, port) + print_studio_stop_hint() return app diff --git a/studio/backend/startup_banner.py b/studio/backend/startup_banner.py index 16b41d484c..2bda4357ba 100644 --- a/studio/backend/startup_banner.py +++ b/studio/backend/startup_banner.py @@ -33,18 +33,49 @@ def print_port_in_use_notice(original_port: int, new_port: int) -> None: print(msg) +def print_studio_stop_hint() -> None: + """Print the trailing stop hint + closing divider. Separate from the main + banner so callers can interleave content (e.g. a reachability check).""" + use_color = stdout_supports_color() + dim = "\033[38;5;245m" + stop_hint_style = "\033[38;5;215;1m" + reset = "\033[0m" + + def style(text: str, code: str) -> str: + return f"{code}{text}{reset}" if use_color else text + + print( + "\n".join( + [ + "", + style( + " To stop Unsloth Studio: press Ctrl+C in this terminal.", + stop_hint_style, + ), + style(" (On macOS this is Control+C, not Command+C.)", dim), + style("─" * 52, dim), + "", + ] + ) + ) + + def print_studio_access_banner( *, port: int, bind_host: str, display_host: str, + include_stop_hint: bool = True, ) -> None: - """Pretty-print URLs after the server is listening (beginner-friendly).""" + """Pretty-print URLs after the server is listening. Set + ``include_stop_hint=False`` to omit the trailing stop block; pair with + :func:`print_studio_stop_hint` after inserting your own content.""" use_color = stdout_supports_color() dim = "\033[38;5;245m" title = "\033[38;5;150m" local_url_style = "\033[38;5;108;1m" secondary = "\033[38;5;109m" + stop_hint_style = "\033[38;5;215;1m" reset = "\033[0m" def style(text: str, code: str) -> str: @@ -116,8 +147,48 @@ def print_studio_access_banner( f" Tip: if you are on this computer, open {tip_url}/ in your browser.", dim, ), - "", ] ) + if loopback_bind and not listen_all: + lines.extend( + [ + "", + style( + " Studio is only reachable on this machine (bound to 127.0.0.1).", + secondary, + ), + style( + " To deploy and access globally:", + secondary, + ), + style( + " 1. press Ctrl+C to stop Studio", + secondary, + ), + style( + f" 2. relaunch with: unsloth studio -H 0.0.0.0 -p {port}", + secondary, + ), + style( + " Only do this on trusted networks -- it exposes the API on every interface.", + secondary, + ), + ] + ) + + if include_stop_hint: + lines.extend( + [ + "", + style( + " To stop Unsloth Studio: press Ctrl+C in this terminal.", + stop_hint_style, + ), + style(" (On macOS this is Control+C, not Command+C.)", dim), + style("─" * 52, dim), + "", + ] + ) + print("\n".join(lines)) diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index aa1d38c4e1..b081c248d0 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -347,6 +347,17 @@ with sync_playwright() as p: except Exception: pass if _form_attempt < 2: + # ERR_NO_BUFFER_SPACE needs the OS to recover socket + # buffers; immediate retry just re-fails. Back off + # 5s then 15s before next attempt. + if "ERR_NO_BUFFER_SPACE" in str(e): + backoff_s = 5 if _form_attempt == 0 else 15 + print( + f"[ui] ENOBUFS detected; sleeping {backoff_s}s " + f"before retry to let OS recover socket buffers...", + flush = True, + ) + time.sleep(backoff_s) # Recovery: replace the page if it died, otherwise the # next loop iteration's page.goto() handles the reload. page = recover_or_replace_page( diff --git a/tests/studio/playwright_extra_ui.py b/tests/studio/playwright_extra_ui.py index dccd2e423d..1f507c9f33 100644 --- a/tests/studio/playwright_extra_ui.py +++ b/tests/studio/playwright_extra_ui.py @@ -214,6 +214,17 @@ with sync_playwright() as p: flush = True, ) if _form_attempt < 2: + # ERR_NO_BUFFER_SPACE needs the OS to recover socket + # buffers; immediate retry just re-fails. Back off + # 5s then 15s before next attempt. + if "ERR_NO_BUFFER_SPACE" in str(e): + backoff_s = 5 if _form_attempt == 0 else 15 + print( + f"[extra-ui] ENOBUFS detected; sleeping {backoff_s}s " + f"before retry to let OS recover socket buffers...", + flush = True, + ) + time.sleep(backoff_s) page = recover_or_replace_page( page, ctx,