From 240efa86f89867a1e503b1bf7b8eaab8f74eb6dd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 May 2026 04:08:53 +0000 Subject: [PATCH] CI(ui): add second Playwright job covering Compare/Recipes/Export/Studio/Settings The first Chat UI Tests step ends by clicking the Shutdown menuitem, which leaves the server dead. So a SECOND Studio is booted on port 18894 in the same job (warm install -- adds ~3-5s) and a second Playwright test exercises the routes the chat UI doesn't touch: 1. /chat?compare=... -- assigns two models, sends 2 prompts, asserts both panes respond (so 4 total new assistant bubbles). 2. /data-recipes -- clicks the first template card, verifies the React-Flow canvas mounts. 3. /export -- in chat-only mode (CI default) asserts the route redirects; in non-chat-only asserts [data-tour='export-cta'] + HF token field exist. 4. /studio -- chat-only redirects, non-chat-only asserts the three tabs (Configure / Current run / History) + [data-tour='studio-*'] anchors exist. 5. Settings dialog -- Cmd/Ctrl-, opens it, cycles through every visible tab (General / Profile / Appearance / Chat / Developer / About), asserts each tab body is non-trivial. Same STRICT=1 mode + soft_fail() pattern as playwright_chat_ui.py. Both Playwright runs' screenshots + studio logs are bundled into the existing studio-ui-smoke-artifacts upload; the artifact name doesn't change. --- .github/workflows/studio-ui-smoke.yml | 58 +++- tests/studio/playwright_extra_ui.py | 376 ++++++++++++++++++++++++++ 2 files changed, 432 insertions(+), 2 deletions(-) create mode 100644 tests/studio/playwright_extra_ui.py diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index c9e608dd2d..cc2070738d 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -162,23 +162,77 @@ jobs: mkdir -p logs/playwright python tests/studio/playwright_chat_ui.py - - name: Stop Studio + - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders) if: always() run: | kill "${STUDIO_PID}" 2>/dev/null || true sleep 2 + # The chat UI test ends by clicking the Shutdown menuitem, which + # leaves the server dead. The extra UI test (Compare / Recipes / + # Export / Studio / Settings) needs a fresh Studio, so we boot a + # second one on a different port. Boot is fast (~3-5s on the + # warm install we already did) so this adds little wall time. + - name: Reset auth + boot Studio for extra UI tests (port 18894) + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18894 \ + > logs/studio_extra.log 2>&1 & + echo "STUDIO_EXTRA_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health on 18894 + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:18894/api/health" > /tmp/health2.json; then + jq -e '.status == "healthy"' /tmp/health2.json && break + fi + sleep 1 + done + jq -e '.status == "healthy"' /tmp/health2.json + + - name: Pass bootstrap pw for extra UI test + run: | + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="CIUiExtra-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + echo "::add-mask::$OLD" + echo "::add-mask::$NEW" + echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV" + echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV" + + - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright + env: + BASE_URL: http://127.0.0.1:18894 + STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }} + STUDIO_NEW_PW: ${{ env.STUDIO_EXTRA_NEW_PW }} + PW_ART_DIR: logs/playwright_extra + STUDIO_UI_STRICT: '1' + GGUF_REPO: ${{ env.GGUF_REPO }} + GGUF_VARIANT: ${{ env.GGUF_VARIANT }} + run: | + mkdir -p logs/playwright_extra + python tests/studio/playwright_extra_ui.py + + - name: Stop second Studio + if: always() + run: | + kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true + sleep 2 + - name: Upload Playwright artifacts # Always upload (not just failure) so a green run's screenshots # are reviewable in the Actions UI -- catches "passed but the # UI is silently broken" regressions that would be invisible - # otherwise. + # otherwise. Both Studio's logs (chat + extra) and BOTH + # Playwright artifact dirs are bundled. if: always() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: studio-ui-smoke-artifacts path: | logs/studio.log + logs/studio_extra.log logs/install.log logs/playwright + logs/playwright_extra retention-days: 7 diff --git a/tests/studio/playwright_extra_ui.py b/tests/studio/playwright_extra_ui.py new file mode 100644 index 0000000000..860c294214 --- /dev/null +++ b/tests/studio/playwright_extra_ui.py @@ -0,0 +1,376 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Studio extra-UI Playwright test. + +Covers the user-visible surfaces that the main chat-UI test doesn't: + + 1. Compare tab (/chat?compare=...): assign two models, send 2 prompts, + assert both panes respond. + 2. Recipes editor (/data-recipes/$recipeId): click first template, + verify the recipe-studio canvas mounts, open + close the Preview + dialog. + 3. Export route (/export): chat-only mode redirects to /chat; + non-chat-only mode shows the export form fields. + 4. Studio training route (/studio): chat-only mode redirects; + non-chat-only verifies the tabs + sections exist. + 5. Settings dialog tabs: Cmd/Ctrl-, opens the dialog; cycle through + each tab and verify it isn't blank. + +The test assumes Studio is freshly booted (must_change_password=true) +on BASE_URL with the bootstrap password in STUDIO_OLD_PW. It does its +own change-password through the UI + model load via /api/inference/load, +matching the pattern in playwright_chat_ui.py. +""" + +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from playwright.sync_api import sync_playwright + +BASE = os.environ["BASE_URL"] +OLD = os.environ["STUDIO_OLD_PW"] +NEW = os.environ.get("STUDIO_NEW_PW", "ExtraUi-NEW-2026!") +GGUF_REPO = os.environ.get("GGUF_REPO", "unsloth/gemma-3-270m-it-GGUF") +GGUF_VARIANT = os.environ.get("GGUF_VARIANT", "UD-Q4_K_XL") +ART_DIR = os.environ.get("PW_ART_DIR", "logs/playwright_extra") +ART = Path(ART_DIR) +ART.mkdir(parents = True, exist_ok = True) +STRICT = os.environ.get("STUDIO_UI_STRICT", "0") == "1" + +_n = [0] +_failed: list[str] = [] + + +def step(s: str) -> None: + print(f"[ui-extra] STEP {s}", flush = True) + + +def info(s: str) -> None: + print(f"[ui-extra] {s}", flush = True) + + +def fail(m: str) -> None: + print(f"[ui-extra] FAIL: {m}", flush = True) + _failed.append(m) + + +def soft_fail(m: str) -> None: + if STRICT: + fail(m) + else: + info(f"WARN (strict-off): {m}") + + +with sync_playwright() as p: + browser = p.chromium.launch(headless = True) + ctx = browser.new_context( + viewport = {"width": 1280, "height": 900}, + reduced_motion = "reduce", + ) + ctx.add_init_script(""" + (function () { + try { + const style = document.createElement("style"); + style.textContent = ` + ::view-transition-old(*), + ::view-transition-new(*) { animation: none !important; } + html, body { pointer-events: auto !important; } + `; + (document.head || document.documentElement).appendChild(style); + } catch (e) {} + })(); + """) + page = ctx.new_page() + page.set_default_timeout(30_000) + page_errors = [] + page.on("pageerror", lambda e: page_errors.append(str(e))) + + def shoot(name: str) -> None: + _n[0] += 1 + page.screenshot( + path = str(ART / f"{_n[0]:02d}-{name}.png"), full_page = True, + ) + + # ───────────────────────────────────────────────────── + # Setup: change-password through the UI + model load. + # ───────────────────────────────────────────────────── + step("setup: change-password + model load") + page.goto(f"{BASE}/change-password") + page.locator("#new-password").wait_for(state = "visible", timeout = 30_000) + page.fill("#new-password", NEW) + page.fill("#confirm-password", NEW) + page.locator('button[type="submit"]').click() + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 60_000) + shoot("01-chat-loaded") + + token = page.evaluate("() => localStorage.getItem('unsloth_auth_token')") + if not token: + fail("no access token after change-password") + sys.exit(1) + load_resp = page.evaluate(f"""async () => {{ + const r = await fetch("{BASE}/api/inference/load", {{ + method: "POST", + headers: {{ + "Authorization": "Bearer {token}", + "Content-Type": "application/json", + }}, + body: JSON.stringify({{ + model_path: "{GGUF_REPO}", + gguf_variant: "{GGUF_VARIANT}", + is_lora: false, + max_seq_length: 2048, + }}), + }}); + return {{status: r.status, body: await r.json()}}; + }}""") + if load_resp["status"] != 200: + fail(f"/api/inference/load -> {load_resp['status']}: {load_resp.get('body')!r}") + sys.exit(1) + info(f"loaded model: {load_resp['body'].get('display_name')}") + page.reload() + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 60_000) + + # Detect chat-only mode: /api/health.chat_only is the source of truth. + # In chat-only mode, /studio + /export redirect to /chat. + health = page.evaluate(f"""async () => {{ + const r = await fetch("{BASE}/api/health"); + return await r.json(); + }}""") + chat_only = bool(health.get("chat_only")) + info(f"chat_only mode: {chat_only}") + + # ───────────────────────────────────────────────────── + # 1. Compare tab. + # ───────────────────────────────────────────────────── + step("Compare tab: send to two panes") + # The Compare nav lives in the sidebar; click it. + compare_nav = page.locator('[data-tour="chat-compare"]').first + if compare_nav.count() == 0: + compare_nav = page.get_by_role( + "button", name = re.compile(r"^\s*Compare\s*$", re.I), + ).first + if compare_nav.count() == 0: + soft_fail("Compare nav not found") + else: + compare_nav.click() + page.wait_for_timeout(1500) + shoot("02-compare-opened") + # Compare view's container. + view = page.locator('[data-tour="chat-compare-view"]').first + if view.count() == 0: + soft_fail("[data-tour='chat-compare-view'] not found after Compare click") + else: + ok_count_before = len(page.locator('[data-role="assistant"]').all()) + # Send first prompt; the shared composer placeholder is + # "Send to both models...". Just type into the composer + # textarea (assistant-ui exposes one in compare-mode too). + cmp_composer = page.get_by_placeholder( + re.compile(r"Send to both models", re.I), + ).first + if cmp_composer.count() == 0: + # Fall back to any visible textarea inside the compare + # view. + cmp_composer = view.locator("textarea").first + if cmp_composer.count() == 0: + soft_fail("compare composer textarea not found") + else: + cmp_composer.click() + cmp_composer.fill("Reply with: A") + page.locator('button[aria-label="Send message"]').first.click() + # Wait for at least 2 NEW assistant bubbles (one per pane). + try: + page.wait_for_function( + """(want) => { + return document.querySelectorAll( + '[data-role="assistant"]' + ).length >= want; + }""", + arg = ok_count_before + 2, + timeout = 180_000, + ) + info("OK Compare: 2 new assistant bubbles after first prompt") + except Exception as exc: + soft_fail(f"Compare: 2 bubbles didn't appear: {exc!r}") + shoot("03-compare-after-A") + + # Send a second prompt -> 4 total new bubbles. + cmp_composer.fill("Reply with: B") + page.locator('button[aria-label="Send message"]').first.click() + try: + page.wait_for_function( + """(want) => { + return document.querySelectorAll( + '[data-role="assistant"]' + ).length >= want; + }""", + arg = ok_count_before + 4, + timeout = 180_000, + ) + info("OK Compare: 4 total new assistant bubbles after second prompt") + except Exception as exc: + soft_fail(f"Compare: 4 bubbles didn't appear: {exc!r}") + shoot("04-compare-after-B") + + # Back to single chat for subsequent steps. + page.goto(f"{BASE}/chat") + composer = page.locator('textarea[aria-label="Message input"]') + composer.wait_for(state = "visible", timeout = 30_000) + + # ───────────────────────────────────────────────────── + # 2. Recipes editor. + # ───────────────────────────────────────────────────── + step("Recipes editor: click first template + Preview dialog") + page.goto(f"{BASE}/data-recipes") + page.wait_for_timeout(1500) + shoot("05-recipes-list") + # Template cards render as