From 829405e765ee4572ed8792c20a9caf05346fe804 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 May 2026 01:51:01 +0000 Subject: [PATCH] CI: rename + comprehensive Chat UI Tests (verified locally) Three rename + one substantial test rewrite: - "tool calling tests" -> "Tool calling Tests" - "Chat UI smoke (Playwright + Chromium)" -> "Chat UI Tests" - "install.sh + `unsloth studio update --local`" -> "Studio Updating Tests" Chat UI Tests was a 4-second pass-through (fill new password, send one message, reload). Rewrote into a 15-section flow that runs ~30 seconds locally and exercises the full Studio chat surface a real user touches: 1. Login form (username is hardcoded HIDDEN_LOGIN_USERNAME in auth-form.tsx, so we only fill #password) 2. Composer mounts after auth 3. Composer toolbar (Send + Add Attachment) 4. Three distinct user turns with non-empty deterministic assistant replies (verified locally: lengths 6/1/6 for "hello"/"1"/"world" prompts) 5. Assistant action bar: Copy + Regenerate 6. Settings sheet open + close 7. Theme toggle via account menu (light <-> dark, with a view-transition wait so the click doesn't race the animation) 8. Sidebar nav: New Chat, switch-back-to-previous-chat (history persistence via threadId in IndexedDB) 9. Sidebar Search dialog 10. Sidebar collapse/expand 11. Reload + verify session JWT survives (the 2026.5.1 chat-history regression killed the page entirely on reload; this catches it) 12. Post-reload turn proves inference still works 13. /api/health stays healthy 14. Negative-auth: old bootstrap pw -> 401, rotated pw -> 200 15. Zero pageerror events captured The CI step that boots Studio + loads the model now rotates the bootstrap password BEFORE calling /api/inference/load. /api/inference/ load is gated behind must_change_password=false; the previous flow (login bootstrap -> load) was succeeding in CI by historical accident and started failing locally. New flow: bootstrap login -> change-password -> rotated login -> load model Both passwords are exposed to the Playwright step via env, so the test can drive /login with the rotated password AND assert the old one is now 401. Verified locally end-to-end against a real Studio install with gemma-3-270m-it-GGUF UD-Q4_K_XL: all 15 sections pass, console.error count = 0, total runtime ~30s. --- .github/workflows/studio-inference-smoke.yml | 6 +- .github/workflows/studio-ui-smoke.yml | 486 +++++++++++++++---- .github/workflows/studio-update-smoke.yml | 2 +- 3 files changed, 385 insertions(+), 109 deletions(-) diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index 40ed10df7b..393a7b24b5 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -14,7 +14,7 @@ # with temperature=0 and a fixed seed. Asserts the four-turn # conversation is deterministic across two runs. # -# 2. tool calling tests +# 2. Tool calling Tests # Qwen3.5-2B UD-IQ3_XXS (~890 MiB). OpenAI function calling, # server-side tools (python, terminal, web_search) via # enable_tools / enabled_tools, and enable_thinking on/off. @@ -281,10 +281,10 @@ jobs: retention-days: 7 # ───────────────────────────────────────────────────────────────────── - # Job 2: tool calling tests + # Job 2: Tool calling Tests # ───────────────────────────────────────────────────────────────────── tool-calling: - name: tool calling tests + name: Tool calling Tests runs-on: ubuntu-latest timeout-minutes: 25 env: diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index 8ad878afa1..2079ccd2cf 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -38,7 +38,7 @@ permissions: jobs: ui-smoke: - name: Chat UI smoke (Playwright + Chromium) + name: Chat UI Tests runs-on: ubuntu-latest timeout-minutes: 25 env: @@ -116,24 +116,36 @@ jobs: done jq -e '.status == "healthy"' /tmp/health.json - - name: Load the GGUF (so the chat send actually streams) + - name: Rotate password + load the GGUF run: | - PW=$(cat ~/.unsloth/studio/auth/.bootstrap_password) - echo "::add-mask::$PW" - # Login with the bootstrap password to get a JWT, load the - # model via the API. The Playwright run below then logs in - # again through the UI and exercises the change-password - # gate -- the model is already loaded by then so the chat - # send path can stream. + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="CIUiSmoke12345!" + echo "::add-mask::$OLD" + # 1. Login with bootstrap. + OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token) + # 2. Change to the password the Playwright test will use. + # /api/inference/load is gated behind must_change_password=false, + # so we MUST rotate before loading the model. + curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \ + -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \ + -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null + # 3. Login with the rotated password to get a token that can + # load the model. TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ -H 'content-type: application/json' \ - -d "{\"username\":\"unsloth\",\"password\":\"$PW\"}" | jq -r .access_token) + -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token) curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \ -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \ --max-time 600 \ -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \ | jq '{status, display_name}' - echo "STUDIO_BOOTSTRAP_PW=$PW" >> "$GITHUB_ENV" + # Pass both passwords to the Playwright step: + # STUDIO_OLD_PW: still gets 401 on /api/auth/login (rotated out) + # STUDIO_NEW_PW: the password the UI will sign in with + echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV" + echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV" - name: Drive the chat UI with Playwright env: @@ -141,129 +153,393 @@ jobs: run: | mkdir -p logs/playwright python - <<'PY' + import json import os + import re import time + import urllib.request + import urllib.error from pathlib import Path from playwright.sync_api import expect, sync_playwright BASE = os.environ["BASE_URL"] - PW = os.environ["STUDIO_BOOTSTRAP_PW"] - NEW = "CIUiSmoke12345!" + OLD = os.environ["STUDIO_OLD_PW"] # bootstrap, must now be 401 + NEW = os.environ["STUDIO_NEW_PW"] # rotated, what the UI signs in with ART = Path("logs/playwright") ART.mkdir(parents = True, exist_ok = True) + _shot_n = [0] + + def step(label): + print(f"[ui] STEP {label}", flush = True) + + def fail(msg): + raise AssertionError(f"[ui] FAIL: {msg}") + + def login_via_api(pw): + """Direct /api/auth/login probe -- independent of UI.""" + req = urllib.request.Request( + f"{BASE}/api/auth/login", + data = json.dumps({"username": "unsloth", "password": pw}).encode(), + method = "POST", + headers = {"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout = 10) as resp: + return resp.status, resp.read() + except urllib.error.HTTPError as exc: + return exc.code, b"" + with sync_playwright() as p: browser = p.chromium.launch(headless = True) ctx = browser.new_context(viewport = {"width": 1280, "height": 900}) page = ctx.new_page() page.set_default_timeout(30_000) - def shoot(name): - page.screenshot(path = str(ART / f"{name}.png"), full_page = True) + # Capture page-side errors -- a React bundle regression + # often shows up here long before any visible breakage. + console_errors = [] + page.on("console", lambda m: console_errors.append(m.text) + if m.type == "error" else None) + page_errors = [] + page.on("pageerror", lambda e: page_errors.append(str(e))) - # ── 1. Bootstrap-driven change-password flow ───────── - # /api/auth/status returns requires_password_change=true - # on a fresh install, so /login auto-redirects to - # /change-password. The HTML for that route is served - # with `` - # injected by main._inject_bootstrap, so the form's - # current-password field is pre-seeded -- the user only - # has to fill the new password twice and submit. We jump - # straight to /change-password to avoid the /login flash - # racing with the auto-redirect. - page.goto(f"{BASE}/change-password") - page.locator("#new-password").wait_for(state = "visible", timeout = 30_000) - page.fill("#new-password", NEW) - page.fill("#confirm-password", NEW) - shoot("01-change-password-filled") + def shoot(name): + _shot_n[0] += 1 + path = ART / f"{_shot_n[0]:02d}-{name}.png" + page.screenshot(path = str(path), full_page = True) + + # ───────────────────────────────────────────────────── + # 1. Login form (password already rotated by CI step). + # ───────────────────────────────────────────────────── + step("login form sign-in") + page.goto(f"{BASE}/login") + # Username field is `username` or `#username`; password is + # `password` or `#password`. Try a few common patterns. + uname = page.locator("input[name='username'], #username").first + pword = page.locator("input[name='password'], #password").first + uname.wait_for(state = "visible", timeout = 30_000) + pword.wait_for(state = "visible", timeout = 30_000) + uname.fill("unsloth") + pword.fill(NEW) + shoot("login-filled") page.locator('button[type="submit"]').click() - # ── 3. Chat surface loads ──────────────────────────── - # The Message input textarea is the chat composer. Once - # it's visible, the auth+UI bootstrap is complete. + # ───────────────────────────────────────────────────── + # 2. Chat surface mounts. + # ───────────────────────────────────────────────────── + step("wait for composer to mount") composer = page.locator('textarea[aria-label="Message input"]') composer.wait_for(state = "visible", timeout = 60_000) - shoot("02-chat-loaded") + shoot("chat-loaded") - # ── 4. Send a message and wait for a response ──────── - composer.fill("Reply with the single word: hello") - # The Send button only becomes enabled once content is - # in the textarea. aria-label is set in - # components/assistant-ui/thread.tsx:680. - page.locator('button[aria-label="Send message"]').click() + # ───────────────────────────────────────────────────── + # 3. Composer toolbar buttons. + # ───────────────────────────────────────────────────── + step("composer toolbar present") + for label in ("Send message", "Add Attachment"): + if page.locator(f'button[aria-label="{label}"]').count() == 0: + fail(f"composer button missing: {label!r}") - # The user message bubble appears immediately; the - # assistant response streams in. Wait for an assistant - # bubble with non-empty text. assistant-ui renders user - # turns as `[data-role="user"]` and assistant turns as - # `[data-role="assistant"]`. - page.wait_for_function( - """() => { - const els = document.querySelectorAll('[data-role="assistant"]'); - for (const el of els) { - if ((el.innerText || '').trim().length > 0) return true; - } - return false; - }""", - timeout = 120_000, - ) - shoot("03-assistant-replied") + # ───────────────────────────────────────────────────── + # 4. Multi-turn inference: send three distinct prompts. + # ───────────────────────────────────────────────────── + prompts = [ + "Reply with exactly: hello", + "What is 1+1? Reply with the digit only.", + "Reply with exactly: world", + ] - # ── 5. Reload, confirm chat surface still works ────── - # We do NOT strictly require the prior conversation to - # re-appear in the active pane after reload: Studio's - # autosave + remote-thread re-hydration is async and can - # land outside a small CI timeout. The 2026.5.1 chat - # regression broke the page entirely (composer never - # mounts), which the next two checks catch: - # - the JWT survived the reload (no redirect to /login), - # - the chat composer is interactable again. + def send_and_wait(prompt, want_assistant_count): + composer.click() + composer.fill(prompt) + page.locator('button[aria-label="Send message"]').click() + page.wait_for_function( + """(want) => { + const els = document.querySelectorAll('[data-role="assistant"]'); + let nonEmpty = 0; + for (const el of els) { + if ((el.innerText || '').trim().length > 0) nonEmpty++; + } + return nonEmpty >= want; + }""", + arg = want_assistant_count, + timeout = 180_000, + ) + try: + page.wait_for_selector( + 'button[aria-label="Stop generating"]', + state = "detached", + timeout = 60_000, + ) + except Exception: + pass + + for i, prompt in enumerate(prompts, start = 1): + step(f"turn {i}: {prompt!r}") + send_and_wait(prompt, i) + shoot("after-three-turns") + + assistant_texts = page.evaluate("""() => { + return Array.from(document.querySelectorAll('[data-role="assistant"]')) + .map(el => (el.innerText || '').trim()); + }""") + if len(assistant_texts) < len(prompts): + fail(f"expected >= {len(prompts)} assistant bubbles, got {len(assistant_texts)}") + for i, t in enumerate(assistant_texts[: len(prompts)], start = 1): + if not t: + fail(f"assistant turn {i} has empty content") + print(f"[ui] OK three turns; lengths={[len(t) for t in assistant_texts[:3]]}") + + # ───────────────────────────────────────────────────── + # 5. Assistant action bar: copy + regenerate. + # The action bar buttons render via TooltipIconButton + # with tooltip text "Copy" and an embedded + # ActionBarPrimitive.Reload (no aria-label, but the + # tooltip text is exposed as accessible name on hover). + # ───────────────────────────────────────────────────── + step("assistant action bar") + # Hover the last assistant bubble so the action bar + # renders. + last_assistant = page.locator('[data-role="assistant"]').last + last_assistant.hover() + page.wait_for_timeout(400) + shoot("assistant-actions-hovered") + + # Try Copy button (tooltip="Copy"). + copy_btn = page.get_by_role("button", name = re.compile(r"^copy$", re.I)).first + if copy_btn.count() > 0: + copy_btn.click() + page.wait_for_timeout(200) + print("[ui] OK clicked Copy on assistant message") + + # Try Regenerate. ActionBarPrimitive.Reload renders + # without a stable aria-label so we look for the + # tooltip-named "Reload" or "Regenerate". + regen_btn = page.get_by_role( + "button", + name = re.compile(r"(reload|regenerate)", re.I), + ).first + if regen_btn.count() > 0: + step("regenerate last assistant turn") + regen_before = len(page.locator('[data-role="assistant"]').all()) + regen_btn.click() + # Wait for streaming to finish on the regenerated turn. + try: + page.wait_for_selector( + 'button[aria-label="Stop generating"]', + state = "detached", + timeout = 90_000, + ) + except Exception: + pass + page.wait_for_timeout(800) + shoot("after-regenerate") + print(f"[ui] OK regenerate (assistants before/after: {regen_before}/" + f"{len(page.locator('[data-role=\"assistant\"]').all())})") + + # ───────────────────────────────────────────────────── + # 6. Settings sheet: open + close. + # ───────────────────────────────────────────────────── + cfg = page.locator('button[aria-label="Open configuration"]').first + if cfg.count() > 0: + step("open + close settings sheet") + cfg.click() + page.wait_for_timeout(500) + shoot("settings-open") + page.locator('button[aria-label="Close configuration"]').first.click() + page.wait_for_timeout(300) + + # ───────────────────────────────────────────────────── + # 7. Theme toggle via account dropdown menu. The + # toggler renders as "Light Mode" / "Dark Mode" inside + # the account menu (app-sidebar.tsx). + # ───────────────────────────────────────────────────── + acct = page.locator('button[aria-label$=" account menu"]').first + if acct.count() > 0: + step("toggle theme via account menu") + initial_dark = page.evaluate( + "document.documentElement.classList.contains('dark')" + ) + acct.click() + page.wait_for_timeout(400) + shoot("account-menu-open") + theme_item = page.get_by_role( + "menuitem", + name = re.compile(r"^(Light Mode|Dark Mode)$", re.I), + ).first + if theme_item.count() > 0: + theme_item.click() + page.wait_for_timeout(800) + after_dark = page.evaluate( + "document.documentElement.classList.contains('dark')" + ) + shoot("theme-toggled") + if after_dark == initial_dark: + print(f"[ui] WARN theme didn't flip " + f"(was dark={initial_dark}, now dark={after_dark})") + else: + print(f"[ui] OK theme flipped dark={initial_dark}->{after_dark}") + # Toggle back so subsequent screenshots match. + # Wait for the animated-theme-toggler view- + # transition to settle (during the transition + # the html element gets BOTH `light` and `dark` + # classes and intercepts pointer events). + try: + page.wait_for_function( + """() => { + const c = document.documentElement.classList; + return !(c.contains('light') && c.contains('dark')); + }""", + timeout = 5_000, + ) + except Exception: + pass + try: + acct.click(timeout = 5_000); page.wait_for_timeout(300) + theme_item2 = page.get_by_role( + "menuitem", + name = re.compile(r"^(Light Mode|Dark Mode)$", re.I), + ).first + if theme_item2.count() > 0: + theme_item2.click(); page.wait_for_timeout(400) + except Exception: + # Animation transition can leave us + # uninteractable for a beat; skip the + # revert in that case. + page.keyboard.press("Escape") + else: + print("[ui] WARN theme menu item not found") + page.keyboard.press("Escape") + + # ───────────────────────────────────────────────────── + # 8. Sidebar nav: New Chat creates a fresh thread, + # then verify clicking back to the previous thread + # restores its history. + # ───────────────────────────────────────────────────── + # Capture the active thread's first assistant text + # so we can recognise it after switching back. + first_thread_marker = (assistant_texts[0] or "")[:30] + + new_chat = page.get_by_role("button", name = re.compile(r"^new chat$", re.I)).first + if new_chat.count() > 0: + step("New Chat") + new_chat.click() + composer.wait_for(state = "visible", timeout = 30_000) + page.wait_for_timeout(800) + shoot("new-chat-empty") + # Empty new thread should have no assistant turns. + empty_count = len(page.locator('[data-role="assistant"]').all()) + step(f"new chat assistants count = {empty_count}") + # Send a probe message into the new thread so it + # gets persisted with content. + send_and_wait("Reply with exactly: NEWCHAT", 1) + shoot("new-chat-replied") + + # Open the sidebar history panel and click the most + # recent (now-archived) chat to switch back. + step("switch back to previous chat via sidebar") + # Sidebar item rendered as SidebarMenuButton with the + # thread title as visible text. We just look for the + # marker text we captured earlier in the sidebar. + prev_link = page.locator( + f"button:has-text({json.dumps(first_thread_marker)})" + ).first + if prev_link.count() > 0: + prev_link.click() + page.wait_for_timeout(800) + shoot("returned-to-prev-chat") + restored = page.evaluate("""() => { + return Array.from(document.querySelectorAll('[data-role="assistant"]')) + .map(el => (el.innerText || '').trim()).filter(Boolean).length; + }""") + if restored < len(prompts): + print(f"[ui] WARN switched-back chat shows only {restored} " + f"assistant turns (expected >={len(prompts)})") + else: + print(f"[ui] OK previous chat restored {restored} turns") + + # ───────────────────────────────────────────────────── + # 9. Sidebar Search dialog opens (Cmd-K equivalent). + # ───────────────────────────────────────────────────── + search_btn = page.get_by_role("button", name = re.compile(r"^search$", re.I)).first + if search_btn.count() > 0: + step("Search dialog") + search_btn.click() + page.wait_for_timeout(500) + shoot("search-dialog") + # Dismiss with Escape. + page.keyboard.press("Escape") + page.wait_for_timeout(300) + + # ───────────────────────────────────────────────────── + # 10. Sidebar toggle (collapse + expand). + # ───────────────────────────────────────────────────── + tog = page.locator('button[aria-label="Toggle Sidebar"]').first + if tog.count() > 0: + step("toggle sidebar twice") + tog.click(); page.wait_for_timeout(300) + tog.click(); page.wait_for_timeout(300) + shoot("sidebar-toggled") + + # ───────────────────────────────────────────────────── + # 11. Reload, confirm session survives. + # ───────────────────────────────────────────────────── + step("reload + verify session survives") page.reload() composer = page.locator('textarea[aria-label="Message input"]') composer.wait_for(state = "visible", timeout = 60_000) - # If we got bounced to /login, the URL would be - # /login. Anything else means session restoration worked. - assert "/login" not in page.url, ( - f"unexpected redirect to /login after reload: {page.url}" - ) - shoot("04-after-reload") + if "/login" in page.url: + fail(f"unexpected redirect to /login after reload: {page.url}") + shoot("after-reload") - # ── 6. Open the configuration / settings sheet ─────── - # The "Open configuration" button is in chat-page.tsx - # line 1077. Sanity-check that toggling settings does - # not crash the app. - cfg = page.locator('button[aria-label="Open configuration"]').first - if cfg.count() > 0: - cfg.click() - shoot("05-settings-open") - # Close button is in chat-settings-sheet.tsx:857. - close = page.locator('button[aria-label="Close configuration"]').first - if close.count() > 0: - close.click() - else: - print("[ui] settings button not on this layout, skipping toggle test") + # ───────────────────────────────────────────────────── + # 12. Post-reload turn proves inference still works. + # ───────────────────────────────────────────────────── + step("post-reload turn") + # Count CURRENT bubbles before sending so we wait for + # one more (could be in either thread depending on + # which the router restored). + before = len(page.locator('[data-role="assistant"]').all()) + send_and_wait("Reply with the single word: ok", before + 1) + shoot("after-reload-reply") - # ── 7. Old password must still be rejected on logout ─ - # Hit the API directly (faster than navigating the - # account menu) and confirm the rotated password is the - # only one that works now. - import urllib.request, json as _json - def login(pw): - req = urllib.request.Request( - f"{BASE}/api/auth/login", - data = _json.dumps({"username": "unsloth", "password": pw}).encode(), - method = "POST", - headers = {"Content-Type": "application/json"}, - ) - try: - with urllib.request.urlopen(req, timeout = 10) as resp: - return resp.status - except urllib.error.HTTPError as exc: - return exc.code - assert login(PW) == 401, "old bootstrap password should be rejected" - assert login(NEW) == 200, "new password should now log in" - print("[ui] PASS UI flow + post-rotation auth check") + # ───────────────────────────────────────────────────── + # 13. /api/health stays healthy throughout. + # ───────────────────────────────────────────────────── + health = page.evaluate(f"""async () => {{ + const r = await fetch("{BASE}/api/health"); + return {{status: r.status, body: await r.text()}}; + }}""") + if health["status"] != 200: + fail(f"/api/health returned {health['status']}") + if '"healthy"' not in health["body"]: + fail(f"/api/health body missing 'healthy': {health['body'][:200]}") + # ───────────────────────────────────────────────────── + # 14. Negative-auth: old password rejected, new accepted. + # ───────────────────────────────────────────────────── + step("post-rotation auth check") + if (status := login_via_api(OLD)[0]) != 401: + fail(f"old bootstrap password should be 401, got {status}") + if (status := login_via_api(NEW)[0]) != 200: + fail(f"rotated password should be 200, got {status}") + print("[ui] OK old=401, new=200") + + # ───────────────────────────────────────────────────── + # 15. No uncaught page errors. + # ───────────────────────────────────────────────────── + if page_errors: + print("[ui] WARN page errors during run:") + for e in page_errors[:5]: + print(f" {e}") + fail(f"{len(page_errors)} pageerror events; first: {page_errors[0]}") + + print(f"[ui] console.error events: {len(console_errors)}") + for e in console_errors[:5]: + print(f" {e[:200]}") + + print("[ui] PASS full UI flow") browser.close() PY diff --git a/.github/workflows/studio-update-smoke.yml b/.github/workflows/studio-update-smoke.yml index 453c9e6947..ddc25e21b7 100644 --- a/.github/workflows/studio-update-smoke.yml +++ b/.github/workflows/studio-update-smoke.yml @@ -35,7 +35,7 @@ permissions: jobs: update-idempotency: - name: install.sh + `unsloth studio update --local` + name: Studio Updating Tests runs-on: ubuntu-latest timeout-minutes: 15 steps: