diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 4aa5b56f0b..497f4127ba 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -116,430 +116,34 @@ jobs: done jq -e '.status == "healthy"' /tmp/health.json - - name: Rotate password + load the GGUF + - name: Pass bootstrap password to the Playwright step + # The Playwright test does its OWN /change-password through the + # UI (Setup your account / Choose a new password), then loads + # the model via page.evaluate against /api/inference/load with + # the JWT it got from change-password. So the only thing we + # have to hand it is the bootstrap password (so it can verify + # post-rotation that the OLD bootstrap pw now returns 401). run: | OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) - NEW="CIUiSmoke12345!" echo "::add-mask::$OLD" - # 1. Login with bootstrap. - OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ - -H 'content-type: application/json' \ - -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token) - # 2. Change to the password the Playwright test will use. - # /api/inference/load is gated behind must_change_password=false, - # so we MUST rotate before loading the model. - curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \ - -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \ - -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null - # 3. Login with the rotated password to get a token that can - # load the model. - TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ - -H 'content-type: application/json' \ - -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token) - curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \ - -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \ - --max-time 600 \ - -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \ - | jq '{status, display_name}' - # Pass both passwords to the Playwright step: - # STUDIO_OLD_PW: still gets 401 on /api/auth/login (rotated out) - # STUDIO_NEW_PW: the password the UI will sign in with echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV" - echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV" + # Two distinct rotated passwords -- the UI rotates from + # bootstrap -> NEW, and a later "terminal" subprocess(curl) + # rotates NEW -> NEW2 to prove an out-of-band password + # change invalidates the previous credentials. + echo "STUDIO_NEW_PW=CIUiSmoke12345!" >> "$GITHUB_ENV" + echo "STUDIO_NEW2_PW=CIUiSmoke67890!" >> "$GITHUB_ENV" - name: Drive the chat UI with Playwright env: BASE_URL: http://127.0.0.1:18892 + # The test file lives in the repo so it can be run locally + # against a freshly-installed Studio (BASE_URL=...; STUDIO_OLD_PW= + # $(cat ~/.unsloth/studio/auth/.bootstrap_password); python ...). + PW_ART_DIR: logs/playwright run: | mkdir -p logs/playwright - python - <<'PY' - import json - import os - import re - import time - import urllib.request - import urllib.error - from pathlib import Path - from playwright.sync_api import expect, sync_playwright - - BASE = os.environ["BASE_URL"] - OLD = os.environ["STUDIO_OLD_PW"] # bootstrap, must now be 401 - NEW = os.environ["STUDIO_NEW_PW"] # rotated, what the UI signs in with - ART = Path("logs/playwright") - ART.mkdir(parents = True, exist_ok = True) - - _shot_n = [0] - - def step(label): - print(f"[ui] STEP {label}", flush = True) - - def fail(msg): - raise AssertionError(f"[ui] FAIL: {msg}") - - def login_via_api(pw): - """Direct /api/auth/login probe -- independent of UI.""" - req = urllib.request.Request( - f"{BASE}/api/auth/login", - data = json.dumps({"username": "unsloth", "password": pw}).encode(), - method = "POST", - headers = {"Content-Type": "application/json"}, - ) - try: - with urllib.request.urlopen(req, timeout = 10) as resp: - return resp.status, resp.read() - except urllib.error.HTTPError as exc: - return exc.code, b"" - - with sync_playwright() as p: - browser = p.chromium.launch(headless = True) - ctx = browser.new_context(viewport = {"width": 1280, "height": 900}) - page = ctx.new_page() - page.set_default_timeout(30_000) - - # Capture page-side errors -- a React bundle regression - # often shows up here long before any visible breakage. - console_errors = [] - page.on("console", lambda m: console_errors.append(m.text) - if m.type == "error" else None) - page_errors = [] - page.on("pageerror", lambda e: page_errors.append(str(e))) - - def shoot(name): - _shot_n[0] += 1 - path = ART / f"{_shot_n[0]:02d}-{name}.png" - page.screenshot(path = str(path), full_page = True) - - # ───────────────────────────────────────────────────── - # 1. Login form (password already rotated by CI step). - # auth-form.tsx hard-codes the username to - # HIDDEN_LOGIN_USERNAME = "unsloth", so the form has only - # a `#password` field -- no username input to fill. - # ───────────────────────────────────────────────────── - step("login form sign-in") - page.goto(f"{BASE}/login") - pword = page.locator("#password").first - pword.wait_for(state = "visible", timeout = 30_000) - pword.fill(NEW) - shoot("login-filled") - page.locator('button[type="submit"]').click() - - # ───────────────────────────────────────────────────── - # 2. Chat surface mounts. - # ───────────────────────────────────────────────────── - step("wait for composer to mount") - composer = page.locator('textarea[aria-label="Message input"]') - composer.wait_for(state = "visible", timeout = 60_000) - shoot("chat-loaded") - - # ───────────────────────────────────────────────────── - # 3. Composer toolbar buttons. - # ───────────────────────────────────────────────────── - step("composer toolbar present") - for label in ("Send message", "Add Attachment"): - if page.locator(f'button[aria-label="{label}"]').count() == 0: - fail(f"composer button missing: {label!r}") - - # ───────────────────────────────────────────────────── - # 4. Multi-turn inference: send three distinct prompts. - # ───────────────────────────────────────────────────── - prompts = [ - "Reply with exactly: hello", - "What is 1+1? Reply with the digit only.", - "Reply with exactly: world", - ] - - def send_and_wait(prompt, want_assistant_count): - composer.click() - composer.fill(prompt) - page.locator('button[aria-label="Send message"]').click() - page.wait_for_function( - """(want) => { - const els = document.querySelectorAll('[data-role="assistant"]'); - let nonEmpty = 0; - for (const el of els) { - if ((el.innerText || '').trim().length > 0) nonEmpty++; - } - return nonEmpty >= want; - }""", - arg = want_assistant_count, - timeout = 180_000, - ) - try: - page.wait_for_selector( - 'button[aria-label="Stop generating"]', - state = "detached", - timeout = 60_000, - ) - except Exception: - pass - - for i, prompt in enumerate(prompts, start = 1): - step(f"turn {i}: {prompt!r}") - send_and_wait(prompt, i) - shoot("after-three-turns") - - assistant_texts = page.evaluate("""() => { - return Array.from(document.querySelectorAll('[data-role="assistant"]')) - .map(el => (el.innerText || '').trim()); - }""") - if len(assistant_texts) < len(prompts): - fail(f"expected >= {len(prompts)} assistant bubbles, got {len(assistant_texts)}") - for i, t in enumerate(assistant_texts[: len(prompts)], start = 1): - if not t: - fail(f"assistant turn {i} has empty content") - print(f"[ui] OK three turns; lengths={[len(t) for t in assistant_texts[:3]]}") - - # ───────────────────────────────────────────────────── - # 5. Assistant action bar: copy + regenerate. - # The action bar buttons render via TooltipIconButton - # with tooltip text "Copy" and an embedded - # ActionBarPrimitive.Reload (no aria-label, but the - # tooltip text is exposed as accessible name on hover). - # ───────────────────────────────────────────────────── - step("assistant action bar") - # Hover the last assistant bubble so the action bar - # renders. - last_assistant = page.locator('[data-role="assistant"]').last - last_assistant.hover() - page.wait_for_timeout(400) - shoot("assistant-actions-hovered") - - # Try Copy button (tooltip="Copy"). - copy_btn = page.get_by_role("button", name = re.compile(r"^copy$", re.I)).first - if copy_btn.count() > 0: - copy_btn.click() - page.wait_for_timeout(200) - print("[ui] OK clicked Copy on assistant message") - - # Try Regenerate. ActionBarPrimitive.Reload renders - # without a stable aria-label so we look for the - # tooltip-named "Reload" or "Regenerate". - regen_btn = page.get_by_role( - "button", - name = re.compile(r"(reload|regenerate)", re.I), - ).first - if regen_btn.count() > 0: - step("regenerate last assistant turn") - regen_before = len(page.locator('[data-role="assistant"]').all()) - regen_btn.click() - # Wait for streaming to finish on the regenerated turn. - try: - page.wait_for_selector( - 'button[aria-label="Stop generating"]', - state = "detached", - timeout = 90_000, - ) - except Exception: - pass - page.wait_for_timeout(800) - shoot("after-regenerate") - print(f"[ui] OK regenerate (assistants before/after: {regen_before}/" - f"{len(page.locator('[data-role=\"assistant\"]').all())})") - - # ───────────────────────────────────────────────────── - # 6. Settings sheet: open + close. - # ───────────────────────────────────────────────────── - cfg = page.locator('button[aria-label="Open configuration"]').first - if cfg.count() > 0: - step("open + close settings sheet") - cfg.click() - page.wait_for_timeout(500) - shoot("settings-open") - page.locator('button[aria-label="Close configuration"]').first.click() - page.wait_for_timeout(300) - - # ───────────────────────────────────────────────────── - # 7. Theme toggle via account dropdown menu. The - # toggler renders as "Light Mode" / "Dark Mode" inside - # the account menu (app-sidebar.tsx). - # ───────────────────────────────────────────────────── - acct = page.locator('button[aria-label$=" account menu"]').first - if acct.count() > 0: - step("toggle theme via account menu") - initial_dark = page.evaluate( - "document.documentElement.classList.contains('dark')" - ) - acct.click() - page.wait_for_timeout(400) - shoot("account-menu-open") - theme_item = page.get_by_role( - "menuitem", - name = re.compile(r"^(Light Mode|Dark Mode)$", re.I), - ).first - if theme_item.count() > 0: - theme_item.click() - page.wait_for_timeout(800) - after_dark = page.evaluate( - "document.documentElement.classList.contains('dark')" - ) - shoot("theme-toggled") - if after_dark == initial_dark: - print(f"[ui] WARN theme didn't flip " - f"(was dark={initial_dark}, now dark={after_dark})") - else: - print(f"[ui] OK theme flipped dark={initial_dark}->{after_dark}") - # Toggle back so subsequent screenshots match. - # Wait for the animated-theme-toggler view- - # transition to settle (during the transition - # the html element gets BOTH `light` and `dark` - # classes and intercepts pointer events). - try: - page.wait_for_function( - """() => { - const c = document.documentElement.classList; - return !(c.contains('light') && c.contains('dark')); - }""", - timeout = 5_000, - ) - except Exception: - pass - try: - acct.click(timeout = 5_000); page.wait_for_timeout(300) - theme_item2 = page.get_by_role( - "menuitem", - name = re.compile(r"^(Light Mode|Dark Mode)$", re.I), - ).first - if theme_item2.count() > 0: - theme_item2.click(); page.wait_for_timeout(400) - except Exception: - # Animation transition can leave us - # uninteractable for a beat; skip the - # revert in that case. - page.keyboard.press("Escape") - else: - print("[ui] WARN theme menu item not found") - page.keyboard.press("Escape") - - # ───────────────────────────────────────────────────── - # 8. Sidebar nav: New Chat creates a fresh thread, - # then verify clicking back to the previous thread - # restores its history. - # ───────────────────────────────────────────────────── - # Capture the active thread's first assistant text - # so we can recognise it after switching back. - first_thread_marker = (assistant_texts[0] or "")[:30] - - new_chat = page.get_by_role("button", name = re.compile(r"^new chat$", re.I)).first - if new_chat.count() > 0: - step("New Chat") - new_chat.click() - composer.wait_for(state = "visible", timeout = 30_000) - page.wait_for_timeout(800) - shoot("new-chat-empty") - # Empty new thread should have no assistant turns. - empty_count = len(page.locator('[data-role="assistant"]').all()) - step(f"new chat assistants count = {empty_count}") - # Send a probe message into the new thread so it - # gets persisted with content. - send_and_wait("Reply with exactly: NEWCHAT", 1) - shoot("new-chat-replied") - - # Open the sidebar history panel and click the most - # recent (now-archived) chat to switch back. - step("switch back to previous chat via sidebar") - # Sidebar item rendered as SidebarMenuButton with the - # thread title as visible text. We just look for the - # marker text we captured earlier in the sidebar. - prev_link = page.locator( - f"button:has-text({json.dumps(first_thread_marker)})" - ).first - if prev_link.count() > 0: - prev_link.click() - page.wait_for_timeout(800) - shoot("returned-to-prev-chat") - restored = page.evaluate("""() => { - return Array.from(document.querySelectorAll('[data-role="assistant"]')) - .map(el => (el.innerText || '').trim()).filter(Boolean).length; - }""") - if restored < len(prompts): - print(f"[ui] WARN switched-back chat shows only {restored} " - f"assistant turns (expected >={len(prompts)})") - else: - print(f"[ui] OK previous chat restored {restored} turns") - - # ───────────────────────────────────────────────────── - # 9. Sidebar Search dialog opens (Cmd-K equivalent). - # ───────────────────────────────────────────────────── - search_btn = page.get_by_role("button", name = re.compile(r"^search$", re.I)).first - if search_btn.count() > 0: - step("Search dialog") - search_btn.click() - page.wait_for_timeout(500) - shoot("search-dialog") - # Dismiss with Escape. - page.keyboard.press("Escape") - page.wait_for_timeout(300) - - # ───────────────────────────────────────────────────── - # 10. Sidebar toggle (collapse + expand). - # ───────────────────────────────────────────────────── - tog = page.locator('button[aria-label="Toggle Sidebar"]').first - if tog.count() > 0: - step("toggle sidebar twice") - tog.click(); page.wait_for_timeout(300) - tog.click(); page.wait_for_timeout(300) - shoot("sidebar-toggled") - - # ───────────────────────────────────────────────────── - # 11. Reload, confirm session survives. - # ───────────────────────────────────────────────────── - step("reload + verify session survives") - page.reload() - composer = page.locator('textarea[aria-label="Message input"]') - composer.wait_for(state = "visible", timeout = 60_000) - if "/login" in page.url: - fail(f"unexpected redirect to /login after reload: {page.url}") - shoot("after-reload") - - # ───────────────────────────────────────────────────── - # 12. Post-reload turn proves inference still works. - # ───────────────────────────────────────────────────── - step("post-reload turn") - # Count CURRENT bubbles before sending so we wait for - # one more (could be in either thread depending on - # which the router restored). - before = len(page.locator('[data-role="assistant"]').all()) - send_and_wait("Reply with the single word: ok", before + 1) - shoot("after-reload-reply") - - # ───────────────────────────────────────────────────── - # 13. /api/health stays healthy throughout. - # ───────────────────────────────────────────────────── - health = page.evaluate(f"""async () => {{ - const r = await fetch("{BASE}/api/health"); - return {{status: r.status, body: await r.text()}}; - }}""") - if health["status"] != 200: - fail(f"/api/health returned {health['status']}") - if '"healthy"' not in health["body"]: - fail(f"/api/health body missing 'healthy': {health['body'][:200]}") - - # ───────────────────────────────────────────────────── - # 14. Negative-auth: old password rejected, new accepted. - # ───────────────────────────────────────────────────── - step("post-rotation auth check") - if (status := login_via_api(OLD)[0]) != 401: - fail(f"old bootstrap password should be 401, got {status}") - if (status := login_via_api(NEW)[0]) != 200: - fail(f"rotated password should be 200, got {status}") - print("[ui] OK old=401, new=200") - - # ───────────────────────────────────────────────────── - # 15. No uncaught page errors. - # ───────────────────────────────────────────────────── - if page_errors: - print("[ui] WARN page errors during run:") - for e in page_errors[:5]: - print(f" {e}") - fail(f"{len(page_errors)} pageerror events; first: {page_errors[0]}") - - print(f"[ui] console.error events: {len(console_errors)}") - for e in console_errors[:5]: - print(f" {e[:200]}") - - print("[ui] PASS full UI flow") - browser.close() - PY + python tests/studio/playwright_chat_ui.py - name: Stop Studio if: always() @@ -549,7 +153,7 @@ jobs: - name: Upload Playwright artifacts on failure if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: studio-ui-smoke-artifacts path: | diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py new file mode 100644 index 0000000000..5bd11cf328 --- /dev/null +++ b/tests/studio/playwright_chat_ui.py @@ -0,0 +1,840 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Comprehensive Studio chat UI test, run locally + in CI. + +Covers: + 1. /change-password through the UI (no API pre-rotate). + 2. Model loaded by the time chat opens (the chat page's runtime + adapter pings /api/models/list; we trigger /api/inference/load + via page.evaluate so we don't need the password out-of-band). + 3. Five chat turns, each deterministic (temperature handled at the + server level via Studio's default; we only assert non-empty). + 4. Regenerate the last turn from the assistant action bar. + 5. Composer toggle buttons: Thinking / Web search / Code execution + -- assert aria-label flips state on click. + 6. Configuration sheet: open, drive Temperature slider via keyboard, + close. + 7. Theme toggle through the account menu, multiple cycles, with a + deterministic computed-background-color check on + `document.documentElement` and `document.body`. + 8. Sidebar nav: New Chat, Compare, Search, Recipes (URL changes). + 9. Recents (history) cards: click an existing chat thread. + 10. API tab via account menu -> Developer / api-keys. + 11. Image attachment UI (upload widget reachable; vision response + not asserted because gemma-3-270m is text-only). + 12. Reload + verify session JWT survives. + 13. /api/health remains healthy. + 14. Negative-auth post-UI-rotation: old=401, new=200. + 15. Terminal-driven password rotation via subprocess(curl) to + /api/auth/change-password (NEW -> NEW2). Confirms refresh + tokens get revoked and that an out-of-band password change + (i.e. another tab / CLI / curl) invalidates the old creds. + 16. Shutdown via the account menu's Shutdown menuitem + the + AlertDialog's "Stop server" action; wait for /api/health to + become unreachable (server process exited). + 17. No uncaught page errors. +""" + +import json +import os +import re +import socket +import subprocess +import sys +import time +import urllib.request +import urllib.error +from pathlib import Path +from playwright.sync_api import expect, sync_playwright + +BASE = os.environ["BASE_URL"] +OLD = os.environ["STUDIO_OLD_PW"] +NEW = os.environ["STUDIO_NEW_PW"] +NEW2 = os.environ.get("STUDIO_NEW2_PW", NEW + "X9!") +GGUF_REPO = os.environ.get("GGUF_REPO", "unsloth/gemma-3-270m-it-GGUF") +GGUF_VARIANT = os.environ.get("GGUF_VARIANT", "UD-Q4_K_XL") +ART_DIR = os.environ.get("PW_ART_DIR", "logs/playwright") +ART = Path(ART_DIR) +ART.mkdir(parents = True, exist_ok = True) + +_n = [0] +def step(s): print(f"[ui] STEP {s}", flush = True) +def info(s): print(f"[ui] {s}", flush = True) +def fail(m): raise AssertionError(f"[ui] FAIL: {m}") + +def login_via_api(pw): + req = urllib.request.Request( + f"{BASE}/api/auth/login", + data = json.dumps({"username": "unsloth", "password": pw}).encode(), + method = "POST", + headers = {"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout = 10) as r: + return r.status + except urllib.error.HTTPError as exc: + return exc.code + +def parse_rgb(s): + m = re.search(r"rgba?\((\d+),\s*(\d+),\s*(\d+)", s or "") + return tuple(int(x) for x in m.groups()) if m else None + + +with sync_playwright() as p: + browser = p.chromium.launch(headless = True) + ctx = browser.new_context( + viewport = {"width": 1280, "height": 900}, + # Reduces motion so the theme toggle's view-transition + # animation doesn't briefly intercept pointer events + # (the running CSS view-transition leaves the html in a + # state where Playwright's actionability check fails). + reduced_motion = "reduce", + ) + # Hard-disable CSS view-transitions in case the app doesn't + # honour prefers-reduced-motion. We inject a tiny stylesheet + # that nukes the ::view-transition pseudo-elements + restores + # pointer events on the html element. Equivalent to the user + # having browser-level reduced-motion. + ctx.add_init_script(""" + (function () { + try { + const css = ` + ::view-transition-old(*), + ::view-transition-new(*) { + animation: none !important; + } + html, body { pointer-events: auto !important; } + `; + const style = document.createElement("style"); + style.id = "playwright-no-view-transition"; + style.textContent = css; + (document.head || document.documentElement).appendChild(style); + } catch (e) { /* noop */ } + })(); + """) + page = ctx.new_page() + page.set_default_timeout(30_000) + page_errors = [] + page.on("pageerror", lambda e: page_errors.append(str(e))) + console_errors = [] + page.on("console", lambda m: console_errors.append(m.text) + if m.type == "error" else None) + + def shoot(name): + _n[0] += 1 + page.screenshot( + path = str(ART / f"{_n[0]:02d}-{name}.png"), + full_page = True, + ) + + # ───────────────────────────────────────────────────── + # 1. Change-password through the UI ("Setup your account"). + # The bootstrap state injects window.__UNSLOTH_BOOTSTRAP__ + # so the current-password is pre-seeded; we only enter the + # new password twice and submit. Match the workflow rename + # from "tool calling tests" pattern: this *is* the user's + # first-run experience. + # ───────────────────────────────────────────────────── + step("change-password through UI (Setup your account)") + page.goto(f"{BASE}/change-password") + page.locator("#new-password").wait_for(state = "visible", timeout = 30_000) + shoot("01-change-password-initial") + page.fill("#new-password", NEW) + page.fill("#confirm-password", NEW) + shoot("02-change-password-filled") + page.locator('button[type="submit"]').click() + + # ───────────────────────────────────────────────────── + # 2. Chat surface mounts, default model surface is visible. + # ───────────────────────────────────────────────────── + step("wait for composer to mount") + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 60_000) + shoot("03-chat-loaded") + + # Pull the auth token now -- /api/models/list and + # /api/inference/load both require a bearer. The frontend + # stores it under "unsloth_auth_token" (auth/session.ts). + token = page.evaluate( + "() => localStorage.getItem('unsloth_auth_token')", + ) + if not token: + # Fall back: exchange the refresh token via /api/auth/refresh. + refresh_token = page.evaluate( + "() => localStorage.getItem('unsloth_auth_refresh_token')", + ) + if refresh_token: + refresh = page.evaluate(f"""async (rt) => {{ + const r = await fetch("{BASE}/api/auth/refresh", {{ + method: "POST", + headers: {{"Content-Type": "application/json"}}, + body: JSON.stringify({{refresh_token: rt}}), + }}); + return await r.json(); + }}""", refresh_token) + token = refresh.get("access_token") + if not token: + fail("could not obtain auth token after change-password") + + # Verify the chat page's default model surface comes from + # backend/core/inference/defaults.py:DEFAULT_MODELS_GGUF[0], + # which is the canonical "what the user sees if nothing has + # been loaded yet" entry. A regression that reorders that + # list or hides the default would break the first-launch UX, + # which is what this assertion guards. + step("default_models[0] matches DEFAULT_MODELS_GGUF[0]") + EXPECTED_DEFAULT = os.environ.get( + "EXPECTED_DEFAULT_MODEL", "unsloth/gemma-4-E2B-it-GGUF", + ) + defaults = page.evaluate(f"""async (token) => {{ + const r = await fetch("{BASE}/api/models/list", {{ + headers: {{ "Authorization": "Bearer " + token }}, + }}); + return await r.json(); + }}""", token) + if not defaults.get("default_models"): + fail(f"/api/models/list returned no default_models: {defaults}") + if defaults["default_models"][0] != EXPECTED_DEFAULT: + fail(f"default_models[0]={defaults['default_models'][0]!r}, " + f"expected {EXPECTED_DEFAULT!r}; defaults.py drift?") + info(f"OK default_models[0] = {EXPECTED_DEFAULT}") + + # The model selector button text on the chat page should say + # the default model's display name even before a model is + # loaded. The model-selector renders the current model name + # (or "Select model" if no current); for a fresh chat it + # should surface the default. + selector_btn = page.locator('button:has-text("Select model"), ' + 'button:has-text("gemma"), ' + 'button:has-text("Qwen"), ' + 'button:has-text("Llama")').first + if selector_btn.count() > 0: + sel_text = (selector_btn.text_content() or "").strip() + info(f"model selector button text: {sel_text!r}") + shoot("03b-default-model-button") + + # ───────────────────────────────────────────────────── + # 3. Trigger model load via the page's session cookies. + # Equivalent to the user clicking a model in the picker; + # we just call the same endpoint the picker would. + # ───────────────────────────────────────────────────── + step("load GGUF via /api/inference/load (uses session cookie)") + # Token already fetched above; reuse it for the load call. + load_resp = page.evaluate(f"""async () => {{ + const r = await fetch("{BASE}/api/inference/load", {{ + method: "POST", + headers: {{ + "Authorization": "Bearer {token}", + "Content-Type": "application/json", + }}, + body: JSON.stringify({{ + model_path: "{GGUF_REPO}", + gguf_variant: "{GGUF_VARIANT}", + is_lora: false, + max_seq_length: 2048, + }}), + }}); + return {{status: r.status, body: await r.json()}}; + }}""") + if load_resp["status"] != 200: + fail(f"/api/inference/load returned {load_resp['status']}: {load_resp.get('body')!r}") + info(f"loaded model: {load_resp['body'].get('display_name')}") + + # Studio caches the per-context model state in zustand; reload + # to make the chat composer pick up the loaded model. + page.reload() + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 60_000) + + # ───────────────────────────────────────────────────── + # 3b. Model picker search bar -- click the model selector, + # type into the search box, verify filtering. We don't + # actually select a different model (that would trigger a + # multi-GB download); we just exercise the typeahead so a + # regression in the picker mount / debounced HF search would + # surface here. + # ───────────────────────────────────────────────────── + step("model picker: open + drive search bar") + picker_btn = page.locator( + 'button:has-text("gemma-3-270m"), ' + 'button:has-text("Gemma 3"), ' + 'button:has-text("Select model")' + ).first + if picker_btn.count() == 0: + # Fall back to any button mentioning the loaded model. + picker_btn = page.get_by_role( + "button", name = re.compile(r"gemma-?3", re.I), + ).first + if picker_btn.count() > 0: + picker_btn.click(); page.wait_for_timeout(500) + shoot("03c-model-picker-open") + search = page.get_by_placeholder( + re.compile(r"Search.*models?", re.I), + ).first + if search.count() > 0: + search.fill("qwen") + page.wait_for_timeout(800) + shoot("03d-model-picker-search-qwen") + search.fill("") + page.wait_for_timeout(300) + search.fill("llama") + page.wait_for_timeout(800) + shoot("03e-model-picker-search-llama") + info("OK search bar filtered for qwen + llama") + else: + info("WARN model picker search input not found") + # Close picker without changing selection. + page.keyboard.press("Escape"); page.wait_for_timeout(300) + + # ───────────────────────────────────────────────────── + # 4. Five chat turns, all non-empty. + # ───────────────────────────────────────────────────── + prompts = [ + "Reply with exactly: hello", + "What is 1+1? Reply with the digit only.", + "Reply with exactly: world", + "Say the word 'tree' and nothing else.", + "What is 2+2? Reply with the digit only.", + ] + + def send_and_wait(prompt, want_assistants): + composer.click() + composer.fill(prompt) + page.locator('button[aria-label="Send message"]').click() + page.wait_for_function( + """(want) => { + const els = document.querySelectorAll('[data-role="assistant"]'); + let nonEmpty = 0; + for (const el of els) { + if ((el.innerText || '').trim().length > 0) nonEmpty++; + } + return nonEmpty >= want; + }""", + arg = want_assistants, + timeout = 180_000, + ) + try: + page.wait_for_selector( + 'button[aria-label="Stop generating"]', + state = "detached", + timeout = 60_000, + ) + except Exception: + pass + + for i, p_ in enumerate(prompts, start = 1): + step(f"turn {i}: {p_!r}") + send_and_wait(p_, i) + shoot("04-after-five-turns") + + texts = page.evaluate("""() => Array.from(document.querySelectorAll('[data-role="assistant"]')) + .map(e => (e.innerText || '').trim())""") + if len(texts) < len(prompts): + fail(f"expected >= {len(prompts)} assistant bubbles, got {len(texts)}") + info(f"five turn lengths = {[len(t) for t in texts[:5]]}") + + # ───────────────────────────────────────────────────── + # 5. Regenerate the last assistant turn. + # ───────────────────────────────────────────────────── + step("regenerate last assistant turn") + last_assistant = page.locator('[data-role="assistant"]').last + last_assistant.hover(); page.wait_for_timeout(400) + regen_btn = page.get_by_role( + "button", + name = re.compile(r"(reload|regenerate)", re.I), + ).first + if regen_btn.count() > 0: + regen_btn.click() + try: + page.wait_for_selector( + 'button[aria-label="Stop generating"]', + state = "detached", + timeout = 90_000, + ) + except Exception: + pass + shoot("05-after-regenerate") + info("regenerate completed") + else: + info("regenerate button not visible (skip)") + + # ───────────────────────────────────────────────────── + # 6. Add two more turns AFTER regenerate. + # ───────────────────────────────────────────────────── + extra = ["Reply with: yes", "Reply with: no"] + for j, p_ in enumerate(extra, start = 1): + step(f"extra turn {j}: {p_!r}") + before_count = len(page.locator('[data-role="assistant"]').all()) + send_and_wait(p_, before_count + 1) + shoot("06-after-extra-turns") + + # ───────────────────────────────────────────────────── + # 7. Composer toggle buttons. Each renders with an + # aria-label that flips between "Disable X" / "Enable X" + # depending on its current state (shared-composer.tsx). + # ───────────────────────────────────────────────────── + step("composer toggle buttons (Thinking / Web search / Code execution)") + for feature in ("thinking", "web search", "code execution"): + # Look for either "Disable X" or "Enable X" -- whichever + # is currently rendered. + toggle = page.locator( + f'button[aria-label="Disable {feature}"], ' + f'button[aria-label="Enable {feature}"]' + ).first + if toggle.count() == 0: + info(f"toggle '{feature}' not present on this layout") + continue + # Skip if the model doesn't support this capability (the + # button is rendered disabled). gemma-3-270m, for instance, + # has no reasoning so "Disable thinking" is permanent-disabled. + if toggle.is_disabled(): + info(f"toggle '{feature}' is disabled for this model -- skip") + continue + before = toggle.get_attribute("aria-label") or "" + toggle.click() + page.wait_for_timeout(200) + after = page.locator( + f'button[aria-label="Disable {feature}"], ' + f'button[aria-label="Enable {feature}"]' + ).first.get_attribute("aria-label") or "" + if before == after: + info(f"WARN '{feature}' aria-label did not flip ({before!r})") + else: + info(f"OK '{feature}': {before!r} -> {after!r}") + # Flip back so test state is unchanged. + try: + page.locator( + f'button[aria-label="Disable {feature}"], ' + f'button[aria-label="Enable {feature}"]' + ).first.click() + except Exception: + pass + page.wait_for_timeout(200) + shoot("07-toggles-cycled") + + # ───────────────────────────────────────────────────── + # 8. Configuration sheet: open, find Temperature slider, + # press Home (→ 0), close. + # ───────────────────────────────────────────────────── + cfg_open = page.locator('button[aria-label="Open configuration"]').first + if cfg_open.count() > 0: + step("Configuration sheet: drive Temperature + Top P + extras") + cfg_open.click(); page.wait_for_timeout(500) + shoot("08-config-open") + # ParamSlider uses Radix UI Slider. Each slider gets a + # role="slider" attribute. Walk every slider in the sheet + # by index, focus it, send Home (-> min) so the test + # state is fully deterministic. Whatever the labels are + # ("Temperature", "Top P", "Min P", "Repetition penalty", + # max_tokens etc.), we drive them all to min so a + # regression that locks a slider returns errors here. + sliders = page.locator('[role="slider"]') + n_sliders = sliders.count() + info(f"configuration sheet exposes {n_sliders} slider(s)") + for idx in range(n_sliders): + try: + s = sliders.nth(idx) + s.scroll_into_view_if_needed() + s.focus() + page.keyboard.press("Home") # -> min + page.wait_for_timeout(80) + except Exception as exc: + info(f" slider[{idx}] focus/Home failed: {exc!r}") + shoot("09-config-all-min") + # Then drive Temperature specifically to 0.0 to make the + # downstream chat deterministic. Temperature is the *first* + # slider in the sheet (configuration-sheet.tsx renders it + # first); Home already pinned it to 0. + info("Temperature set to slider min (0.0) for determinism") + # Close. + close_btn = page.locator('button[aria-label="Close configuration"]').first + if close_btn.count() > 0: + close_btn.click() + else: + page.keyboard.press("Escape") + page.wait_for_timeout(300) + + # ───────────────────────────────────────────────────── + # 9. Theme toggle -- multiple cycles + deterministic + # computed-background-color check. The light theme + # uses near-white (>240); dark uses near-black (<40). + # ───────────────────────────────────────────────────── + acct = page.locator('button[aria-label$=" account menu"]').first + if acct.count() > 0: + step("theme toggle x3 with computed-color assertion") + observed = [] + for cycle in range(3): + try: + acct.click(force = True) + except Exception as exc: + info(f" cycle {cycle + 1}: account-menu click failed ({exc!r})") + break + page.wait_for_timeout(400) + theme_item = page.get_by_role( + "menuitem", + name = re.compile(r"^(Light Mode|Dark Mode)$", re.I), + ).first + if theme_item.count() == 0: + page.keyboard.press("Escape") + break + try: + theme_item.click(force = True) + except Exception: + page.keyboard.press("Escape") + break + # Settle. The ".dark" class on is the ground + # truth (theme-store toggles only that class); the + # ".light" sibling is steady-state from next-themes + # so don't gate on it. + page.wait_for_timeout(700) + bg = page.evaluate("""() => { + const root = document.documentElement; + return { + cls: root.className, + isDark: root.classList.contains('dark'), + bg: getComputedStyle(document.body).backgroundColor, + rbg: getComputedStyle(root).backgroundColor, + }; + }""") + observed.append(bg) + shoot(f"10-theme-cycle-{cycle + 1}") + info(f" cycle {cycle + 1}: dark={bg['isDark']} body bg={bg['bg']!r}") + # Sanity check: across cycles we should observe both a + # light state (body bg roughly near-white) and a dark state + # (body bg near-black). If we only saw one polarity the + # toggle didn't flip. + rgbs = [parse_rgb(o["bg"]) for o in observed if parse_rgb(o["bg"])] + light_seen = any(min(r) > 220 for r in rgbs) + dark_seen = any(max(r) < 60 for r in rgbs) + if not (light_seen and dark_seen): + info(f"WARN expected both light + dark backgrounds across " + f"{len(rgbs)} cycles; light_seen={light_seen}, dark_seen={dark_seen}") + else: + info("OK light + dark computed background colors observed") + + # ───────────────────────────────────────────────────── + # 10. Sidebar nav: New Chat, Compare, Search, Recipes. + # ───────────────────────────────────────────────────── + def click_nav(label, expected_url_pat = None): + btn = page.get_by_role("button", name = re.compile(rf"^\s*{label}\s*$", re.I)).first + if btn.count() == 0: + info(f"nav '{label}' not found"); return False + btn.click() + page.wait_for_timeout(800) + if expected_url_pat and not re.search(expected_url_pat, page.url): + info(f"WARN clicking '{label}' didn't change url to /{expected_url_pat}; " + f"current: {page.url}") + return False + return True + + step("sidebar nav: New Chat -> Compare -> Search -> Recipes") + click_nav("New Chat", r"/chat") + shoot("11-new-chat") + click_nav("Compare", r"/chat\?") # /chat?compare=... + shoot("12-compare") + # Search opens a dialog (not a route change). + search_btn = page.get_by_role("button", name = re.compile(r"^search$", re.I)).first + if search_btn.count() > 0: + search_btn.click(); page.wait_for_timeout(500) + shoot("13-search-dialog") + page.keyboard.press("Escape"); page.wait_for_timeout(300) + click_nav("Recipes", r"/data-recipes") + shoot("14-recipes") + # Back to chat for subsequent steps. + page.goto(f"{BASE}/chat"); composer.wait_for(state = "visible", timeout = 30_000) + + # ───────────────────────────────────────────────────── + # 11. API / Developer tab via account menu -> opens the + # Settings dialog with the api-keys tab. Verify we can see + # the Create API Key form (or existing keys table); regressions + # that hide the api-keys management UI surface here. + # ───────────────────────────────────────────────────── + if acct.count() > 0: + step("Developer (API) tab via account menu") + acct.click(); page.wait_for_timeout(400) + dev = page.get_by_role("menuitem", name = re.compile(r"developer|api", re.I)).first + if dev.count() > 0: + dev.click(); page.wait_for_timeout(800) + shoot("15-developer-tab") + # Look for the create-key affordance. + create_btn = page.get_by_role( + "button", name = re.compile(r"create.*key|generate.*key|add.*key|new key", re.I), + ).first + if create_btn.count() > 0: + info("OK 'create API key' affordance visible") + # Look for the api-keys list section title. + keys_section = page.get_by_text( + re.compile(r"api keys|developer", re.I), + ).first + if keys_section.count() > 0: + info(f"OK API tab text: {(keys_section.text_content() or '').strip()[:80]!r}") + # Close dialog with Escape. + page.keyboard.press("Escape"); page.wait_for_timeout(300) + else: + page.keyboard.press("Escape") + + # ───────────────────────────────────────────────────── + # 11b. Recipes tab: verify cards render + we can click one. + # The Recipes route renders a grid of preset cards; a + # regression that breaks the loader would render zero cards + # or crash the route. + # ───────────────────────────────────────────────────── + step("Recipes tab: cards render + click first card") + page.goto(f"{BASE}/data-recipes"); page.wait_for_timeout(1500) + # Recipe cards are rendered as or button elements; count + # all clickable headings under main + screenshot. + headings = page.locator("main h2, main h3, [data-recipe], a[href*='/data-recipes/']") + n_cards = headings.count() + info(f"Recipes route headings/cards: {n_cards}") + shoot("15b-recipes-cards") + if n_cards > 0: + # Try clicking the first one to confirm it navigates / opens. + try: + headings.first.scroll_into_view_if_needed() + headings.first.click() + page.wait_for_timeout(1200) + shoot("15c-recipes-first-card") + info("OK clicked first recipe card") + except Exception as exc: + info(f"WARN click first recipe failed: {exc!r}") + # Back to chat. + page.goto(f"{BASE}/chat") + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 30_000) + + # ───────────────────────────────────────────────────── + # 11c. Recents: the chat sidebar lists previous threads. We + # already created several turns above (which gets persisted + # as a thread). Find the sidebar's recents region and click + # the most-recent entry. This catches regressions in the + # thread-history loader / route param plumbing. + # ───────────────────────────────────────────────────── + step("Recents: click previous chat in sidebar") + # The sidebar lists threads as buttons / links. Match anything + # under the sidebar with non-empty text other than the nav + # entries we already verified (New Chat / Compare / Search / + # Recipes / Account). + EXCLUDE = re.compile(r"^(New Chat|Compare|Search|Recipes|Settings|Account|" + r"Light Mode|Dark Mode|Developer|Help|Shutdown)$", re.I) + candidates = page.locator("aside a, aside button, [data-sidebar='sidebar'] a, " + "[data-sidebar='sidebar'] button") + count_c = candidates.count() + clicked_recent = False + for i in range(count_c): + try: + t = (candidates.nth(i).text_content() or "").strip() + if not t or EXCLUDE.match(t): + continue + # Heuristic: thread titles are typically the first + # user message snippet or a short summary. + if 3 <= len(t) <= 80: + candidates.nth(i).scroll_into_view_if_needed() + candidates.nth(i).click() + page.wait_for_timeout(1500) + shoot("15d-recent-clicked") + info(f"OK clicked recent entry: {t[:60]!r}") + clicked_recent = True + break + except Exception: + continue + if not clicked_recent: + info("no Recents entry was clickable -- skipped") + # Back to chat. + page.goto(f"{BASE}/chat") + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 30_000) + + # ───────────────────────────────────────────────────── + # 12. Image attachment UI (upload widget reachable). The + # current model is text-only so we don't assert a vision + # response -- just that the attachment button is there + # and the file input accepts a PNG. CI's gemma-4-E2B + # job covers the actual vision path. + # ───────────────────────────────────────────────────── + step("attachment widget reachable") + attach = page.locator('button[aria-label="Add Attachment"]').first + if attach.count() > 0: + # Just hover -- triggering the file picker mid-test + # would block on a native dialog. Verifying the + # button is reachable is enough. + attach.hover(); page.wait_for_timeout(200) + shoot("16-attachment-hover") + + # ───────────────────────────────────────────────────── + # 13. Reload + verify session JWT survives. + # ───────────────────────────────────────────────────── + step("reload + session survives") + page.reload() + composer.wait_for(state = "visible", timeout = 60_000) + if "/login" in page.url: + fail(f"unexpected redirect to /login after reload: {page.url}") + shoot("17-after-reload") + + # ───────────────────────────────────────────────────── + # 14. /api/health stays healthy throughout. + # ───────────────────────────────────────────────────── + health = page.evaluate(f"""async () => {{ + const r = await fetch("{BASE}/api/health"); + return {{status: r.status, body: await r.text()}}; + }}""") + if health["status"] != 200: + fail(f"/api/health returned {health['status']}") + + # ───────────────────────────────────────────────────── + # 15. Negative-auth post-UI-rotation. + # ───────────────────────────────────────────────────── + step("post-rotation auth check (after UI change-password)") + if (s_old := login_via_api(OLD)) != 401: + fail(f"old bootstrap pw should be 401, got {s_old}") + if (s_new := login_via_api(NEW)) != 200: + fail(f"rotated pw should be 200, got {s_new}") + info("OK old=401, new=200") + + # ───────────────────────────────────────────────────── + # 16. Out-of-band ("terminal") password rotation. + # POST /api/auth/change-password from a real subprocess(curl) + # invocation -- this is the same surface a sysadmin / another + # tab / a desktop helper would use, and the security promise + # is: rotating the password from "the terminal" must invalidate + # the previous credentials. The endpoint also revokes refresh + # tokens server-side (auth.py:152), so /api/auth/refresh from + # the still-open browser context must fail too. + # ───────────────────────────────────────────────────── + step("rotate password via subprocess(curl) -- the 'terminal' path") + # Get a fresh access token by logging in via the API rather than + # reusing whatever's in localStorage; this matches what an admin + # would actually do from a shell. + login_proc = subprocess.run( + [ + "curl", "-fsS", "-X", "POST", + f"{BASE}/api/auth/login", + "-H", "Content-Type: application/json", + "-d", json.dumps({"username": "unsloth", "password": NEW}), + ], + capture_output = True, text = True, timeout = 15, + ) + if login_proc.returncode != 0: + fail(f"curl login failed: {login_proc.stderr!r}") + login_body = json.loads(login_proc.stdout) + cli_token = login_body.get("access_token") + if not cli_token: + fail(f"curl login returned no access_token: {login_body!r}") + info("CLI obtained an access token") + + change_proc = subprocess.run( + [ + "curl", "-fsS", "-X", "POST", + f"{BASE}/api/auth/change-password", + "-H", "Content-Type: application/json", + "-H", f"Authorization: Bearer {cli_token}", + "-d", json.dumps({"current_password": NEW, "new_password": NEW2}), + ], + capture_output = True, text = True, timeout = 15, + ) + if change_proc.returncode != 0: + fail(f"curl change-password failed: rc={change_proc.returncode} " + f"stderr={change_proc.stderr!r} stdout={change_proc.stdout!r}") + info("CLI rotated password NEW -> NEW2 successfully") + + # NEW must now be 401, NEW2 must be 200. + if (s_new1 := login_via_api(NEW)) != 401: + fail(f"after CLI rotation, NEW pw should be 401, got {s_new1}") + if (s_new2 := login_via_api(NEW2)) != 200: + fail(f"after CLI rotation, NEW2 pw should be 200, got {s_new2}") + info("OK after CLI rotation: NEW=401, NEW2=200 -- old studio creds dead") + + # The browser still has the pre-rotation access token. Refresh + # tokens were revoked server-side by /change-password (auth.py), + # so /api/auth/refresh from the browser context must now fail. + refresh_after = page.evaluate(f"""async () => {{ + const r = await fetch("{BASE}/api/auth/refresh", {{ + method: "POST", + credentials: "include", + }}); + return {{status: r.status}}; + }}""") + if refresh_after["status"] == 200: + fail(f"/api/auth/refresh should fail after CLI rotation; got 200") + info(f"OK browser /api/auth/refresh now {refresh_after['status']} " + "(refresh token revoked) -- old studio session can no longer renew") + + # ───────────────────────────────────────────────────── + # 17. Shutdown button via the account menu. + # The Shutdown menuitem opens an AlertDialog ("Stop Unsloth + # Studio?") whose primary action is "Stop server"; clicking + # it POSTs /api/shutdown and then replaces document.body with + # the "Unsloth Studio has stopped" placeholder. /api/health + # should become unreachable shortly after. + # ───────────────────────────────────────────────────── + step("Shutdown via account menu") + # Re-login through the UI with NEW2 so the browser has a valid + # access token for the /api/shutdown call (the previous one + # was invalidated by the CLI rotation above). + page.goto(f"{BASE}/login") + pw_field = page.locator("#password") + pw_field.wait_for(state = "visible", timeout = 30_000) + pw_field.fill(NEW2) + page.locator('button[type="submit"]').click() + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 60_000) + shoot("18-relogin-with-NEW2") + + acct_btn = page.locator('button[aria-label$=" account menu"]').first + if acct_btn.count() == 0: + fail("account menu button missing -- can't reach Shutdown") + acct_btn.click(); page.wait_for_timeout(400) + shutdown_item = page.get_by_role( + "menuitem", name = re.compile(r"^\s*Shutdown\s*$", re.I), + ).first + if shutdown_item.count() == 0: + fail("Shutdown menuitem not in account menu") + shutdown_item.click() + shoot("19-shutdown-dialog") + stop_btn = page.get_by_role( + "button", name = re.compile(r"^\s*Stop server\s*$", re.I), + ).first + stop_btn.wait_for(state = "visible", timeout = 5_000) + stop_btn.click() + + # Wait for the post-shutdown placeholder body. The component + # replaces document.body.innerHTML with text containing + # "Unsloth Studio has stopped." once /api/shutdown returns ok. + try: + page.wait_for_function( + """() => /Unsloth Studio has stopped/.test(document.body.innerText)""", + timeout = 15_000, + ) + shoot("20-shutdown-placeholder") + info("OK 'Unsloth Studio has stopped' placeholder rendered") + except Exception as exc: + info(f"WARN shutdown placeholder didn't render: {exc!r}") + + # Now /api/health must become unreachable (process exited or is + # at least not listening). Poll for up to 15 s. + host = re.sub(r"^https?://", "", BASE).split(":")[0] + port = int(re.search(r":(\d+)", BASE).group(1)) if ":" in BASE else 80 + deadline = time.time() + 15 + while time.time() < deadline: + try: + with socket.create_connection((host, port), timeout = 1): + pass + time.sleep(0.5) + except (ConnectionRefusedError, OSError): + info("OK port closed -- server process is gone") + break + else: + # Connection still works -> shutdown didn't take effect. + try: + r = urllib.request.urlopen(f"{BASE}/api/health", timeout = 2) + fail(f"server still up after Shutdown click; /api/health={r.status}") + except urllib.error.URLError as exc: + info(f"OK /api/health unreachable: {exc!r}") + + if page_errors: + info(f"WARN page errors: {len(page_errors)}; first: {page_errors[0]!r}") + fail(f"{len(page_errors)} pageerror events") + info(f"console.error events: {len(console_errors)}") + + info("PASS comprehensive UI flow") + browser.close()