unsloth/.github/workflows/studio-ui-smoke.yml
Daniel Han 2f219615ad CI(ui): drop nonexistent username locator (auth form is password-only)
studio/frontend/src/features/auth/components/auth-form.tsx hard-codes
the login username to HIDDEN_LOGIN_USERNAME = "unsloth"; the only
visible input is #password. The previous Playwright step waited 30s
for `input[name='username'], #username` and timed out on every CI run.

I caught this locally and patched the test script during validation
but didn't bring the fix back to the workflow file -- this commit
applies it. Wait for #password only, fill the rotated password, click
submit. Verified locally end-to-end against a fresh Studio.
2026-05-07 02:10:14 +00:00

559 lines
28 KiB
YAML

# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
# End-to-end Studio chat UI smoke via Playwright + Chromium against a
# headless Linux runner. Boots Studio with the smallest GGUF
# (gemma-3-270m-it UD-Q4_K_XL, ~254 MiB), drives the actual frontend
# bundle, and asserts the full bootstrap-password / change-password /
# send-message / persist-on-reload journey works end to end.
#
# This is the only workflow that catches regressions in the wiring
# between the React frontend and the FastAPI backend, e.g. assistant-ui
# version drift, /api/auth response shape changes, runtime-provider
# regressions, or chat-history persistence breaking. Backend-only and
# frontend-only CI happily pass while the actual user-visible UI is
# broken (cf. the 2026.5.1 chat-history release).
name: Studio UI CI
on:
pull_request:
paths:
- 'studio/**'
- 'unsloth/**'
- 'unsloth_cli/**'
- 'install.sh'
- 'pyproject.toml'
- '.github/workflows/studio-ui-smoke.yml'
push:
branches: [main, pip]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
ui-smoke:
name: Chat UI Tests
runs-on: ubuntu-latest
timeout-minutes: 25
env:
GGUF_REPO: unsloth/gemma-3-270m-it-GGUF
GGUF_VARIANT: UD-Q4_K_XL
GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf
STUDIO_PORT: '18892'
HF_HOME: ${{ github.workspace }}/hf-cache
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
- name: Linux deps
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libcurl4-openssl-dev libssl-dev jq
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: '22'
cache: 'npm'
cache-dependency-path: studio/frontend/package-lock.json
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: '3.12'
cache: 'pip'
- name: Cache HF_HOME for ${{ env.GGUF_REPO }}
id: cache-hf
uses: actions/cache@v4
with:
path: hf-cache
key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1
- name: Prime HF_HOME with the GGUF
if: steps.cache-hf.outputs.cache-hit != 'true'
run: |
python -m pip install --upgrade huggingface_hub hf_transfer
mkdir -p hf-cache
HF_HUB_ENABLE_HF_TRANSFER=1 \
hf download "$GGUF_REPO" "$GGUF_FILE"
- name: Install Studio (--local, --no-torch)
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
mkdir -p logs
set -o pipefail
bash install.sh --local --no-torch 2>&1 | tee logs/install.log
- name: Install Playwright + Chromium
run: |
pip install 'playwright>=1.45'
# --with-deps installs the OS-level runtime libs Chromium
# needs (libnss3, libxkbcommon, etc.). About 30 s on a
# warm runner.
python -m playwright install --with-deps chromium
- name: Reset auth + boot Studio
run: |
unsloth studio reset-password
mkdir -p logs
UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \
> logs/studio.log 2>&1 &
echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
- name: Wait for /api/health
run: |
for i in $(seq 1 90); do
if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then
jq -e '.status == "healthy"' /tmp/health.json && break
fi
sleep 1
done
jq -e '.status == "healthy"' /tmp/health.json
- name: Rotate password + load the GGUF
run: |
OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password)
NEW="CIUiSmoke12345!"
echo "::add-mask::$OLD"
# 1. Login with bootstrap.
OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
-H 'content-type: application/json' \
-d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token)
# 2. Change to the password the Playwright test will use.
# /api/inference/load is gated behind must_change_password=false,
# so we MUST rotate before loading the model.
curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \
-H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \
-d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null
# 3. Login with the rotated password to get a token that can
# load the model.
TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \
-H 'content-type: application/json' \
-d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token)
curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \
-H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \
--max-time 600 \
-d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \
| jq '{status, display_name}'
# Pass both passwords to the Playwright step:
# STUDIO_OLD_PW: still gets 401 on /api/auth/login (rotated out)
# STUDIO_NEW_PW: the password the UI will sign in with
echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV"
echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV"
- name: Drive the chat UI with Playwright
env:
BASE_URL: http://127.0.0.1:18892
run: |
mkdir -p logs/playwright
python - <<'PY'
import json
import os
import re
import time
import urllib.request
import urllib.error
from pathlib import Path
from playwright.sync_api import expect, sync_playwright
BASE = os.environ["BASE_URL"]
OLD = os.environ["STUDIO_OLD_PW"] # bootstrap, must now be 401
NEW = os.environ["STUDIO_NEW_PW"] # rotated, what the UI signs in with
ART = Path("logs/playwright")
ART.mkdir(parents = True, exist_ok = True)
_shot_n = [0]
def step(label):
print(f"[ui] STEP {label}", flush = True)
def fail(msg):
raise AssertionError(f"[ui] FAIL: {msg}")
def login_via_api(pw):
"""Direct /api/auth/login probe -- independent of UI."""
req = urllib.request.Request(
f"{BASE}/api/auth/login",
data = json.dumps({"username": "unsloth", "password": pw}).encode(),
method = "POST",
headers = {"Content-Type": "application/json"},
)
try:
with urllib.request.urlopen(req, timeout = 10) as resp:
return resp.status, resp.read()
except urllib.error.HTTPError as exc:
return exc.code, b""
with sync_playwright() as p:
browser = p.chromium.launch(headless = True)
ctx = browser.new_context(viewport = {"width": 1280, "height": 900})
page = ctx.new_page()
page.set_default_timeout(30_000)
# Capture page-side errors -- a React bundle regression
# often shows up here long before any visible breakage.
console_errors = []
page.on("console", lambda m: console_errors.append(m.text)
if m.type == "error" else None)
page_errors = []
page.on("pageerror", lambda e: page_errors.append(str(e)))
def shoot(name):
_shot_n[0] += 1
path = ART / f"{_shot_n[0]:02d}-{name}.png"
page.screenshot(path = str(path), full_page = True)
# ─────────────────────────────────────────────────────
# 1. Login form (password already rotated by CI step).
# auth-form.tsx hard-codes the username to
# HIDDEN_LOGIN_USERNAME = "unsloth", so the form has only
# a `#password` field -- no username input to fill.
# ─────────────────────────────────────────────────────
step("login form sign-in")
page.goto(f"{BASE}/login")
pword = page.locator("#password").first
pword.wait_for(state = "visible", timeout = 30_000)
pword.fill(NEW)
shoot("login-filled")
page.locator('button[type="submit"]').click()
# ─────────────────────────────────────────────────────
# 2. Chat surface mounts.
# ─────────────────────────────────────────────────────
step("wait for composer to mount")
composer = page.locator('textarea[aria-label="Message input"]')
composer.wait_for(state = "visible", timeout = 60_000)
shoot("chat-loaded")
# ─────────────────────────────────────────────────────
# 3. Composer toolbar buttons.
# ─────────────────────────────────────────────────────
step("composer toolbar present")
for label in ("Send message", "Add Attachment"):
if page.locator(f'button[aria-label="{label}"]').count() == 0:
fail(f"composer button missing: {label!r}")
# ─────────────────────────────────────────────────────
# 4. Multi-turn inference: send three distinct prompts.
# ─────────────────────────────────────────────────────
prompts = [
"Reply with exactly: hello",
"What is 1+1? Reply with the digit only.",
"Reply with exactly: world",
]
def send_and_wait(prompt, want_assistant_count):
composer.click()
composer.fill(prompt)
page.locator('button[aria-label="Send message"]').click()
page.wait_for_function(
"""(want) => {
const els = document.querySelectorAll('[data-role="assistant"]');
let nonEmpty = 0;
for (const el of els) {
if ((el.innerText || '').trim().length > 0) nonEmpty++;
}
return nonEmpty >= want;
}""",
arg = want_assistant_count,
timeout = 180_000,
)
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
state = "detached",
timeout = 60_000,
)
except Exception:
pass
for i, prompt in enumerate(prompts, start = 1):
step(f"turn {i}: {prompt!r}")
send_and_wait(prompt, i)
shoot("after-three-turns")
assistant_texts = page.evaluate("""() => {
return Array.from(document.querySelectorAll('[data-role="assistant"]'))
.map(el => (el.innerText || '').trim());
}""")
if len(assistant_texts) < len(prompts):
fail(f"expected >= {len(prompts)} assistant bubbles, got {len(assistant_texts)}")
for i, t in enumerate(assistant_texts[: len(prompts)], start = 1):
if not t:
fail(f"assistant turn {i} has empty content")
print(f"[ui] OK three turns; lengths={[len(t) for t in assistant_texts[:3]]}")
# ─────────────────────────────────────────────────────
# 5. Assistant action bar: copy + regenerate.
# The action bar buttons render via TooltipIconButton
# with tooltip text "Copy" and an embedded
# ActionBarPrimitive.Reload (no aria-label, but the
# tooltip text is exposed as accessible name on hover).
# ─────────────────────────────────────────────────────
step("assistant action bar")
# Hover the last assistant bubble so the action bar
# renders.
last_assistant = page.locator('[data-role="assistant"]').last
last_assistant.hover()
page.wait_for_timeout(400)
shoot("assistant-actions-hovered")
# Try Copy button (tooltip="Copy").
copy_btn = page.get_by_role("button", name = re.compile(r"^copy$", re.I)).first
if copy_btn.count() > 0:
copy_btn.click()
page.wait_for_timeout(200)
print("[ui] OK clicked Copy on assistant message")
# Try Regenerate. ActionBarPrimitive.Reload renders
# without a stable aria-label so we look for the
# tooltip-named "Reload" or "Regenerate".
regen_btn = page.get_by_role(
"button",
name = re.compile(r"(reload|regenerate)", re.I),
).first
if regen_btn.count() > 0:
step("regenerate last assistant turn")
regen_before = len(page.locator('[data-role="assistant"]').all())
regen_btn.click()
# Wait for streaming to finish on the regenerated turn.
try:
page.wait_for_selector(
'button[aria-label="Stop generating"]',
state = "detached",
timeout = 90_000,
)
except Exception:
pass
page.wait_for_timeout(800)
shoot("after-regenerate")
print(f"[ui] OK regenerate (assistants before/after: {regen_before}/"
f"{len(page.locator('[data-role=\"assistant\"]').all())})")
# ─────────────────────────────────────────────────────
# 6. Settings sheet: open + close.
# ─────────────────────────────────────────────────────
cfg = page.locator('button[aria-label="Open configuration"]').first
if cfg.count() > 0:
step("open + close settings sheet")
cfg.click()
page.wait_for_timeout(500)
shoot("settings-open")
page.locator('button[aria-label="Close configuration"]').first.click()
page.wait_for_timeout(300)
# ─────────────────────────────────────────────────────
# 7. Theme toggle via account dropdown menu. The
# toggler renders as "Light Mode" / "Dark Mode" inside
# the account menu (app-sidebar.tsx).
# ─────────────────────────────────────────────────────
acct = page.locator('button[aria-label$=" account menu"]').first
if acct.count() > 0:
step("toggle theme via account menu")
initial_dark = page.evaluate(
"document.documentElement.classList.contains('dark')"
)
acct.click()
page.wait_for_timeout(400)
shoot("account-menu-open")
theme_item = page.get_by_role(
"menuitem",
name = re.compile(r"^(Light Mode|Dark Mode)$", re.I),
).first
if theme_item.count() > 0:
theme_item.click()
page.wait_for_timeout(800)
after_dark = page.evaluate(
"document.documentElement.classList.contains('dark')"
)
shoot("theme-toggled")
if after_dark == initial_dark:
print(f"[ui] WARN theme didn't flip "
f"(was dark={initial_dark}, now dark={after_dark})")
else:
print(f"[ui] OK theme flipped dark={initial_dark}->{after_dark}")
# Toggle back so subsequent screenshots match.
# Wait for the animated-theme-toggler view-
# transition to settle (during the transition
# the html element gets BOTH `light` and `dark`
# classes and intercepts pointer events).
try:
page.wait_for_function(
"""() => {
const c = document.documentElement.classList;
return !(c.contains('light') && c.contains('dark'));
}""",
timeout = 5_000,
)
except Exception:
pass
try:
acct.click(timeout = 5_000); page.wait_for_timeout(300)
theme_item2 = page.get_by_role(
"menuitem",
name = re.compile(r"^(Light Mode|Dark Mode)$", re.I),
).first
if theme_item2.count() > 0:
theme_item2.click(); page.wait_for_timeout(400)
except Exception:
# Animation transition can leave us
# uninteractable for a beat; skip the
# revert in that case.
page.keyboard.press("Escape")
else:
print("[ui] WARN theme menu item not found")
page.keyboard.press("Escape")
# ─────────────────────────────────────────────────────
# 8. Sidebar nav: New Chat creates a fresh thread,
# then verify clicking back to the previous thread
# restores its history.
# ─────────────────────────────────────────────────────
# Capture the active thread's first assistant text
# so we can recognise it after switching back.
first_thread_marker = (assistant_texts[0] or "")[:30]
new_chat = page.get_by_role("button", name = re.compile(r"^new chat$", re.I)).first
if new_chat.count() > 0:
step("New Chat")
new_chat.click()
composer.wait_for(state = "visible", timeout = 30_000)
page.wait_for_timeout(800)
shoot("new-chat-empty")
# Empty new thread should have no assistant turns.
empty_count = len(page.locator('[data-role="assistant"]').all())
step(f"new chat assistants count = {empty_count}")
# Send a probe message into the new thread so it
# gets persisted with content.
send_and_wait("Reply with exactly: NEWCHAT", 1)
shoot("new-chat-replied")
# Open the sidebar history panel and click the most
# recent (now-archived) chat to switch back.
step("switch back to previous chat via sidebar")
# Sidebar item rendered as SidebarMenuButton with the
# thread title as visible text. We just look for the
# marker text we captured earlier in the sidebar.
prev_link = page.locator(
f"button:has-text({json.dumps(first_thread_marker)})"
).first
if prev_link.count() > 0:
prev_link.click()
page.wait_for_timeout(800)
shoot("returned-to-prev-chat")
restored = page.evaluate("""() => {
return Array.from(document.querySelectorAll('[data-role="assistant"]'))
.map(el => (el.innerText || '').trim()).filter(Boolean).length;
}""")
if restored < len(prompts):
print(f"[ui] WARN switched-back chat shows only {restored} "
f"assistant turns (expected >={len(prompts)})")
else:
print(f"[ui] OK previous chat restored {restored} turns")
# ─────────────────────────────────────────────────────
# 9. Sidebar Search dialog opens (Cmd-K equivalent).
# ─────────────────────────────────────────────────────
search_btn = page.get_by_role("button", name = re.compile(r"^search$", re.I)).first
if search_btn.count() > 0:
step("Search dialog")
search_btn.click()
page.wait_for_timeout(500)
shoot("search-dialog")
# Dismiss with Escape.
page.keyboard.press("Escape")
page.wait_for_timeout(300)
# ─────────────────────────────────────────────────────
# 10. Sidebar toggle (collapse + expand).
# ─────────────────────────────────────────────────────
tog = page.locator('button[aria-label="Toggle Sidebar"]').first
if tog.count() > 0:
step("toggle sidebar twice")
tog.click(); page.wait_for_timeout(300)
tog.click(); page.wait_for_timeout(300)
shoot("sidebar-toggled")
# ─────────────────────────────────────────────────────
# 11. Reload, confirm session survives.
# ─────────────────────────────────────────────────────
step("reload + verify session survives")
page.reload()
composer = page.locator('textarea[aria-label="Message input"]')
composer.wait_for(state = "visible", timeout = 60_000)
if "/login" in page.url:
fail(f"unexpected redirect to /login after reload: {page.url}")
shoot("after-reload")
# ─────────────────────────────────────────────────────
# 12. Post-reload turn proves inference still works.
# ─────────────────────────────────────────────────────
step("post-reload turn")
# Count CURRENT bubbles before sending so we wait for
# one more (could be in either thread depending on
# which the router restored).
before = len(page.locator('[data-role="assistant"]').all())
send_and_wait("Reply with the single word: ok", before + 1)
shoot("after-reload-reply")
# ─────────────────────────────────────────────────────
# 13. /api/health stays healthy throughout.
# ─────────────────────────────────────────────────────
health = page.evaluate(f"""async () => {{
const r = await fetch("{BASE}/api/health");
return {{status: r.status, body: await r.text()}};
}}""")
if health["status"] != 200:
fail(f"/api/health returned {health['status']}")
if '"healthy"' not in health["body"]:
fail(f"/api/health body missing 'healthy': {health['body'][:200]}")
# ─────────────────────────────────────────────────────
# 14. Negative-auth: old password rejected, new accepted.
# ─────────────────────────────────────────────────────
step("post-rotation auth check")
if (status := login_via_api(OLD)[0]) != 401:
fail(f"old bootstrap password should be 401, got {status}")
if (status := login_via_api(NEW)[0]) != 200:
fail(f"rotated password should be 200, got {status}")
print("[ui] OK old=401, new=200")
# ─────────────────────────────────────────────────────
# 15. No uncaught page errors.
# ─────────────────────────────────────────────────────
if page_errors:
print("[ui] WARN page errors during run:")
for e in page_errors[:5]:
print(f" {e}")
fail(f"{len(page_errors)} pageerror events; first: {page_errors[0]}")
print(f"[ui] console.error events: {len(console_errors)}")
for e in console_errors[:5]:
print(f" {e[:200]}")
print("[ui] PASS full UI flow")
browser.close()
PY
- name: Stop Studio
if: always()
run: |
kill "${STUDIO_PID}" 2>/dev/null || true
sleep 2
- name: Upload Playwright artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: studio-ui-smoke-artifacts
path: |
logs/studio.log
logs/install.log
logs/playwright
retention-days: 7