From 00e863ed8ca36671dc312524ecbfd94df99dc9bb Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 7 May 2026 09:59:12 +0000 Subject: [PATCH] CI(windows): four Windows Studio CI workflows on free windows-latest + Linux chat-UI fix Adds four Windows counterparts to the existing Mac Studio jobs, all on the free windows-latest runner (4 vCPU / 16 GB / 14 GB SSD; no premium SKU). Mirrors the Mac coverage 1:1 in name and assertion shape so the PR-status grid reads "Mac Studio * = Windows Studio *": studio-windows-ui-smoke.yml -> "Windows Studio UI CI" studio-windows-inference-smoke.yml -> "Windows Studio GGUF CI" (3 jobs) studio-windows-update-smoke.yml -> "Windows Studio Update CI" studio-windows-api-smoke.yml -> "Windows Studio API CI" Key Windows differences vs the Mac mirrors: * runs-on: windows-latest (free public runner) * defaults.run.shell: bash so curl / jq / heredoc steps go through Git Bash (windows-latest's default shell is pwsh) * Install step uses pwsh + ./install.ps1 --local --no-torch (NOT bash install.sh; install.sh has no Windows branch and would hit apt-get / brew calls). install.ps1 is Studio's documented Windows installer and is exercised by release-desktop.yml today. * Asserter looks for bin-win-cpu-x64 (the prebuilt that windows-latest, no GPU, hits via studio/install_llama_prebuilt.py line 1272). Source-build fallback is rejected as a Studio bug. * setup-python: drop cache:'pip' across all four (install.ps1 + setup.ps1 use uv; setup-python's post-step otherwise fatal-errors with "Cache folder path is retrieved for pip but doesn't exist"). * api-smoke: do NOT pin STUDIO_AUTH_DIR (Mac mirror hardcodes /Users/runner/...). studio_api_smoke.py defaults to Path.home()/'.unsloth'/'studio'/'auth' which resolves correctly on every OS. * inference-smoke: drop the Linux-only `ss -tln` diagnostic line. No code changes to install.ps1, setup.ps1, install_llama_prebuilt.py, or unsloth_cli/commands/studio.py -- Windows is already fully wired in those (~30 host.is_windows branches in the prebuilt installer + three sys.platform=='win32' branches in the Studio CLI). Also fixes the Linux Chat UI Tests "extra turn" timeout (run 25487410101 / job 74786523982). The send_and_wait predicate used non-empty assistant bubble count vs a baseline. When gemma-3-270m emitted an empty turn (legitimate model output), the empty bubble counted toward total but NOT toward the non-empty baseline, and the next turn's wait expected nonempty >= baseline + 1 forever -- never satisfied. Refactor: * Snapshot TOTAL bubble count before send (proves new placeholder rendered, regardless of content). * Wait for Send-button-attached AND Stop-button-detached as the "previous turn finished" signal. * Treat empty bubbles as legitimate model output, not test failure. * Add page.on('response') listener for /v1/chat/completions and log status distribution + 4xx count after the 5-turn loop, so a flake is debuggable from the CI log without artifact spelunking. --- .../workflows/studio-windows-api-smoke.yml | 152 ++++ .../studio-windows-inference-smoke.yml | 849 ++++++++++++++++++ .github/workflows/studio-windows-ui-smoke.yml | 231 +++++ .../workflows/studio-windows-update-smoke.yml | 157 ++++ tests/studio/playwright_chat_ui.py | 118 ++- 5 files changed, 1476 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/studio-windows-api-smoke.yml create mode 100644 .github/workflows/studio-windows-inference-smoke.yml create mode 100644 .github/workflows/studio-windows-ui-smoke.yml create mode 100644 .github/workflows/studio-windows-update-smoke.yml diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml new file mode 100644 index 0000000000..32e901c083 --- /dev/null +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Windows counterpart to studio-api-smoke.yml / studio-mac-api-smoke.yml. +# Same tests/studio/studio_api_smoke.py exercise (CORS hardening, auth +# state machine, JWT expiry, API key lifecycle, /v1/models / +# /v1/embeddings / /v1/responses, endpoint-by-endpoint auth audit) but +# on the FREE windows-latest runner. The file-mode hardening section +# (Section 6) is Linux-only and short-circuits on non-POSIX; the rest +# is platform-portable. + +name: Windows Studio API CI + +on: + pull_request: + paths: + - 'studio/**' + - 'unsloth/**' + - 'unsloth_cli/**' + - 'install.ps1' + - 'pyproject.toml' + - 'tests/studio/**' + - '.github/workflows/studio-windows-api-smoke.yml' + push: + branches: [main, pip] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + api-smoke: + name: Studio API & Auth Tests + runs-on: windows-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + env: + GGUF_REPO: unsloth/gemma-3-270m-it-GGUF + GGUF_VARIANT: UD-Q4_K_XL + GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18895' + HF_HOME: ${{ github.workspace }}/hf-cache + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: studio/frontend/package-lock.json + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + id: cache-hf + uses: actions/cache@v4 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + + - name: Prime HF_HOME with the GGUF + if: steps.cache-hf.outputs.cache-hit != 'true' + run: | + python -m pip install --upgrade huggingface_hub hf_transfer + mkdir -p hf-cache + HF_HUB_ENABLE_HF_TRANSFER=1 \ + hf download "$GGUF_REPO" "$GGUF_FILE" + + - name: Install Studio (--local, --no-torch) + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + & ./install.ps1 --local --no-torch 2>&1 | Tee-Object -FilePath logs/install.log + + - name: Assert install.ps1 used the Windows llama.cpp prebuilt + run: | + if grep -q "falling back to source build" logs/install.log; then + echo "::error::install.ps1 fell back to source-build llama.cpp on Windows. Studio must install the prebuilt llama-bNNNN-bin-win-cpu-x64 on Windows." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + + - name: Install pyjwt for the JWT-expiry forge test + run: python -m pip install 'pyjwt>=2.6' + + - name: Reset auth + boot Studio (API-only) + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ + > logs/studio.log 2>&1 & + echo "STUDIO_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then + jq -e '.status == "healthy"' /tmp/health.json && break + fi + sleep 1 + done + jq -e '.status == "healthy"' /tmp/health.json + + - name: Pass bootstrap password + rotated targets to the test + run: | + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + NEW2="ApiSmoke-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + echo "::add-mask::$OLD" + echo "::add-mask::$NEW" + echo "::add-mask::$NEW2" + echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV" + echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV" + echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV" + + - name: Run Studio API & Auth tests + # Do NOT pin STUDIO_AUTH_DIR here. The Mac/Linux mirrors + # hardcode runner-specific paths (/Users/runner/..., + # /home/runner/...), but on Windows the path is + # C:\Users\runneradmin\.unsloth\studio\auth and varies by + # runner image. studio_api_smoke.py defaults to + # Path.home()/".unsloth"/"studio"/"auth" when the env is + # unset, which is correct on every OS. + env: + BASE_URL: http://127.0.0.1:18895 + run: python tests/studio/studio_api_smoke.py + + - name: Stop Studio + if: always() + run: | + kill "${STUDIO_PID}" 2>/dev/null || true + sleep 2 + + - name: Upload API smoke logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: windows-studio-api-smoke-log + path: | + logs/install.log + logs/studio.log + retention-days: 7 diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml new file mode 100644 index 0000000000..839411c68a --- /dev/null +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -0,0 +1,849 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Three end-to-end smoke jobs that boot a freshly-installed Studio and +# exercise the surfaces real users hit through the OpenAI / Anthropic +# SDKs and curl, on the FREE windows-latest runner. Each job picks the +# smallest model that exercises the behaviour under test, primes +# HF_HOME via actions/cache, and shares the install.ps1 --local +# --no-torch bootstrap. +# +# 1. OpenAI, Anthropic API tests +# gemma-3-270m-it UD-Q4_K_XL (~254 MiB). +# 2. Tool calling Tests +# Qwen3.5-2B UD-Q4_K_XL (~890 MiB). +# 3. JSON, images +# gemma-4-E2B-it UD-Q4_K_XL + mmproj-F16 (~3.4 GiB total). +# Within the 14 GB windows-latest SSD budget. + +name: Windows Studio GGUF CI + +on: + pull_request: + paths: + - 'studio/**' + - 'unsloth/**' + - 'unsloth_cli/**' + - 'install.ps1' + - 'pyproject.toml' + - '.github/workflows/studio-windows-inference-smoke.yml' + push: + branches: [main, pip] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # ───────────────────────────────────────────────────────────────────── + # Job 1: OpenAI, Anthropic API tests + # ───────────────────────────────────────────────────────────────────── + openai-anthropic: + name: OpenAI, Anthropic API tests + runs-on: windows-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + env: + GGUF_REPO: unsloth/gemma-3-270m-it-GGUF + GGUF_VARIANT: UD-Q4_K_XL + GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18888' + HF_HOME: ${{ github.workspace }}/hf-cache + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: studio/frontend/package-lock.json + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + id: cache-hf + uses: actions/cache@v4 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + + - name: Prime HF_HOME with the GGUF + if: steps.cache-hf.outputs.cache-hit != 'true' + run: | + python -m pip install --upgrade huggingface_hub hf_transfer + mkdir -p hf-cache + HF_HUB_ENABLE_HF_TRANSFER=1 \ + hf download "$GGUF_REPO" "$GGUF_FILE" + + - name: Install Studio (--local, --no-torch) + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + & ./install.ps1 --local --no-torch 2>&1 | Tee-Object -FilePath logs/install.log + + - name: Assert install.ps1 used the Windows llama.cpp prebuilt + run: | + if grep -q "falling back to source build" logs/install.log; then + echo "::error::install.ps1 fell back to source-build llama.cpp on Windows. Studio must install the prebuilt llama-bNNNN-bin-win-cpu-x64 on Windows." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + + - name: Install OpenAI + Anthropic Python SDKs + run: python -m pip install 'openai>=1.50' 'anthropic>=0.40' + + - name: Reset auth + boot Studio (API-only) + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ + > logs/studio.log 2>&1 & + echo "STUDIO_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then + jq -e '.status == "healthy"' /tmp/health.json + exit 0 + fi + sleep 1 + done + echo "Studio did not become healthy in 180s" + tail -200 logs/studio.log + exit 1 + + - name: Password rotation (old must fail, new must work) + run: | + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="CIRotated-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')" + echo "::add-mask::$OLD" + echo "::add-mask::$NEW" + OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token) + [ -n "$OLD_TOKEN" ] && [ "$OLD_TOKEN" != "null" ] || { echo "bootstrap login failed"; exit 1; } + curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \ + -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \ + -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null + OLD_STATUS=$(curl -s -o /dev/null -w '%{http_code}' \ + -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}") + if [ "$OLD_STATUS" != "401" ]; then + echo "::error::Login with old password returned $OLD_STATUS, expected 401" + exit 1 + fi + NEW_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token) + [ -n "$NEW_TOKEN" ] && [ "$NEW_TOKEN" != "null" ] || { echo "new login failed"; exit 1; } + echo "TOKEN=$NEW_TOKEN" >> "$GITHUB_ENV" + echo "password rotation OK (old=401, new=200)" + + - name: Load the GGUF (HF repo + variant, served from HF_HOME cache) + run: | + curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \ + -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \ + --max-time 600 \ + -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \ + | jq '{status, display_name, is_gguf, context_length}' + + - name: Multi-turn determinism via OpenAI + Anthropic SDKs + env: + BASE_URL: http://127.0.0.1:18888 + run: | + python - <<'PY' + import json + import os + from openai import OpenAI + from anthropic import Anthropic + + BASE = os.environ["BASE_URL"] + KEY = os.environ["TOKEN"] + SEED = 3407 + + PROMPTS = [ + "What is 1+1?", + "What did I ask before?", + "What is the capital of France?", + "Repeat the city name", + ] + + def run_openai(): + client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY) + history, replies = [], [] + for prompt in PROMPTS: + history.append({"role": "user", "content": prompt}) + resp = client.chat.completions.create( + model = "default", + messages = history, + temperature = 0.0, + max_tokens = 80, + seed = SEED, + extra_body = {"enable_thinking": False}, + ) + text = resp.choices[0].message.content or "" + replies.append(text) + history.append({"role": "assistant", "content": text}) + return replies + + def run_anthropic(): + client = Anthropic( + base_url = BASE, + api_key = "unused", + default_headers = {"Authorization": f"Bearer {KEY}"}, + ) + history, replies = [], [] + for prompt in PROMPTS: + history.append({"role": "user", "content": prompt}) + msg = client.messages.create( + model = "default", + max_tokens = 80, + messages = history, + temperature = 0.0, + extra_body = {"seed": SEED, "enable_thinking": False}, + ) + text = "".join(b.text for b in msg.content if getattr(b, "type", None) == "text") + replies.append(text) + history.append({"role": "assistant", "content": text}) + return replies + + for label, runner in (("openai", run_openai), ("anthropic", run_anthropic)): + first = runner() + second = runner() + for i, (a, b) in enumerate(zip(first, second), start = 1): + print(f"[{label} turn {i}] {a!r}") + assert a, f"{label}: empty turn {i} response" + assert a == b, ( + f"{label} non-deterministic at turn {i} with temperature=0.0:\n" + f" run1: {a!r}\n run2: {b!r}" + ) + joined = " ".join(first).lower() + assert "1" in first[0], f"{label}: turn-1 answer should contain '1', got {first[0]!r}" + assert "paris" in joined, f"{label}: expected 'paris' somewhere in the four-turn transcript: {first}" + print(f"[{label}] OK -- 4 turns, run1 == run2, history grounded") + PY + + - name: Stop Studio + if: always() + run: | + kill "${STUDIO_PID}" 2>/dev/null || true + sleep 2 + + - name: Upload logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: windows-openai-anthropic-log + path: | + logs/studio.log + logs/install.log + retention-days: 7 + + # ───────────────────────────────────────────────────────────────────── + # Job 2: Tool calling Tests + # ───────────────────────────────────────────────────────────────────── + tool-calling: + name: Tool calling Tests + runs-on: windows-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + env: + GGUF_REPO: unsloth/Qwen3.5-2B-GGUF + # Mirror the Mac job's variant choice (UD-Q4_K_XL). On the + # CPU-only windows-latest runner the smaller IQ3_XXS quant + # behaves OK, but keeping parity with the Mac job means one HF + # asset shared across runners. + GGUF_VARIANT: UD-Q4_K_XL + GGUF_FILE: Qwen3.5-2B-UD-Q4_K_XL.gguf + STUDIO_PORT: '18898' + HF_HOME: ${{ github.workspace }}/hf-cache + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: studio/frontend/package-lock.json + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + id: cache-hf + uses: actions/cache@v4 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + + - name: Prime HF_HOME with the GGUF + if: steps.cache-hf.outputs.cache-hit != 'true' + run: | + python -m pip install --upgrade huggingface_hub hf_transfer + mkdir -p hf-cache + HF_HUB_ENABLE_HF_TRANSFER=1 \ + hf download "$GGUF_REPO" "$GGUF_FILE" + + - name: Install Studio (--local, --no-torch) + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + & ./install.ps1 --local --no-torch 2>&1 | Tee-Object -FilePath logs/install.log + + - name: Assert install.ps1 used the Windows llama.cpp prebuilt + run: | + if grep -q "falling back to source build" logs/install.log; then + echo "::error::install.ps1 fell back to source-build llama.cpp on Windows. Studio must install the prebuilt llama-bNNNN-bin-win-cpu-x64 on Windows." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + + - name: Reset auth + boot Studio (API-only, default tool policy) + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ + > logs/studio.log 2>&1 & + echo "STUDIO_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health, log in, change password, load model + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then + jq -e '.status == "healthy"' /tmp/health.json && break + fi + sleep 1 + done + jq -e '.status == "healthy"' /tmp/health.json + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="CITool-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')" + echo "::add-mask::$OLD" + echo "::add-mask::$NEW" + OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token) + curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \ + -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \ + -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null + TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token) + echo "API_KEY=$TOKEN" >> "$GITHUB_ENV" + curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \ + -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \ + --max-time 600 \ + -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \ + | jq '{status, display_name}' + + - name: Tool calling, server-side tools, thinking on/off + env: + BASE_URL: http://127.0.0.1:18898 + run: | + python - <<'PY' + import json + import os + import urllib.request + + BASE = os.environ["BASE_URL"] + KEY = os.environ["API_KEY"] + SEED = 3407 + # Same temperature shim as the Mac job. Small Qwen3.5-2B + # quants can degenerate at temperature=0; a small non-zero + # temperature with a fixed seed keeps the test deterministic + # while escaping the trap. + TEMP = 0.2 + + def post(path, body, *, timeout = 240): + data = json.dumps(body).encode() + req = urllib.request.Request( + f"{BASE}{path}", + data = data, + method = "POST", + headers = { + "Authorization": f"Bearer {KEY}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(req, timeout = timeout) as resp: + return resp.status, json.loads(resp.read().decode()) + + def post_sse(path, body, *, timeout = 600): + body = {**body, "stream": True} + data = json.dumps(body).encode() + req = urllib.request.Request( + f"{BASE}{path}", + data = data, + method = "POST", + headers = { + "Authorization": f"Bearer {KEY}", + "Content-Type": "application/json", + }, + ) + parts = [] + with urllib.request.urlopen(req, timeout = timeout) as resp: + for raw in resp: + line = raw.decode().strip() + if not line.startswith("data: "): + continue + payload = line[6:] + if payload == "[DONE]": + break + try: + chunk = json.loads(payload) + except json.JSONDecodeError: + continue + for choice in chunk.get("choices", []): + delta = choice.get("delta", {}) or {} + if delta.get("content"): + parts.append(delta["content"]) + return "".join(parts) + + # ── 1. Standard OpenAI function calling ────────────────────── + weather_tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + + status, data = post("/v1/chat/completions", { + "messages": [{"role": "user", "content": "What is the weather in Paris?"}], + "tools": [weather_tool], + "tool_choice": "required", + "stream": False, + "temperature": TEMP, + "seed": SEED, + "max_tokens": 600, + }) + assert status == 200, f"tool call status {status}: {data}" + choice = data["choices"][0] + tool_calls = (choice.get("message") or {}).get("tool_calls") or [] + if tool_calls: + tc = tool_calls[0] + assert tc["function"]["name"] == "get_weather", ( + f"unexpected tool name: {tc['function']['name']!r}" + ) + args = json.loads(tc["function"]["arguments"]) + assert args.get("city"), f"missing city arg: {args}" + print(f"[tools] PASS function calling -> {tc['function']['name']}({args}) finish={choice.get('finish_reason')!r}") + else: + print( + f"[tools] WARN function calling: no tool_calls (finish_reason=" + f"{choice.get('finish_reason')!r}); HTTP path OK, model output drift." + ) + + # ── 2. Server-side python tool ─────────────────────────────── + content = post_sse("/v1/chat/completions", { + "messages": [{"role": "user", "content": "What is 123 * 456? Use the python tool to compute it and tell me the number."}], + "enable_tools": True, + "enabled_tools": ["python"], + "session_id": "ci-tool-calling-py", + "temperature": TEMP, + "seed": SEED, + "max_tokens": 600, + }) + if "56088" in content or "56,088" in content: + print(f"[tools] PASS python tool ({len(content)} chars, found 56088)") + else: + assert content, "python tool: SSE stream empty" + print( + f"[tools] WARN python tool: SSE OK ({len(content)} chars) but " + f"model didn't return 56088 -- model output drift" + ) + + # ── 3. Server-side bash (terminal) tool ────────────────────── + # On Windows the terminal tool resolves to the system shell + # (cmd.exe wrapper) and `echo hello-bash-tool` works the same + # way it does on POSIX. The model still has to choose to + # invoke the tool; assert non-empty SSE if it doesn't. + content = post_sse("/v1/chat/completions", { + "messages": [{"role": "user", "content": "Use the terminal tool to run `echo hello-bash-tool` and tell me the exact output."}], + "enable_tools": True, + "enabled_tools": ["terminal"], + "session_id": "ci-tool-calling-bash", + "temperature": TEMP, + "seed": SEED, + "max_tokens": 600, + }) + if "hello-bash-tool" in content: + print(f"[tools] PASS terminal tool ({len(content)} chars)") + else: + assert content, "terminal tool: SSE stream empty" + print( + f"[tools] WARN terminal tool: SSE OK ({len(content)} chars) but " + f"model didn't echo 'hello-bash-tool' -- model output drift" + ) + + # ── 4. Server-side web_search tool ─────────────────────────── + # DuckDuckGo can be flaky from CI runners; only assert that + # the SSE stream opens and yields any data. + try: + content = post_sse("/v1/chat/completions", { + "messages": [{"role": "user", "content": "Search the web for 'unsloth ai github' and summarise."}], + "enable_tools": True, + "enabled_tools": ["web_search"], + "session_id": "ci-tool-calling-web", + "temperature": TEMP, + "seed": SEED, + "max_tokens": 400, + }) + print(f"[tools] PASS web_search stream ({len(content)} chars)") + except Exception as exc: + print(f"[tools] WARN web_search probe failed (non-blocking): {exc}") + + # ── 5. Thinking on / off ───────────────────────────────────── + def thinking_call(enable): + status, data = post("/v1/chat/completions", { + "messages": [{"role": "user", "content": "Briefly: is 17 prime?"}], + "stream": False, + "enable_thinking": enable, + "temperature": TEMP, + "seed": SEED, + "max_tokens": 300, + }) + assert status == 200 + msg = data["choices"][0]["message"] + raw = (msg.get("content") or "") + (msg.get("reasoning_content") or "") + return raw + + on_text = thinking_call(True) + off_text = thinking_call(False) + had_think_on = ("" in on_text) or len(on_text) > 80 + if not had_think_on: + print( + f"[tools] WARN enable_thinking=True produced no thinking signal: " + f"{on_text[:200]!r}" + ) + assert "" not in off_text, ( + f"enable_thinking=False but still present: {off_text!r}" + ) + print(f"[tools] PASS thinking on/off (on={len(on_text)} chars, off={len(off_text)} chars)") + PY + + - name: Stop Studio + if: always() + run: | + kill "${STUDIO_PID}" 2>/dev/null || true + sleep 2 + + - name: Upload logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: windows-tool-calling-log + path: | + logs/studio.log + logs/install.log + retention-days: 7 + + # ───────────────────────────────────────────────────────────────────── + # Job 3: JSON, images + # ───────────────────────────────────────────────────────────────────── + json-images: + name: JSON, images + runs-on: windows-latest + timeout-minutes: 35 + defaults: + run: + shell: bash + env: + GGUF_REPO: unsloth/gemma-4-E2B-it-GGUF + GGUF_VARIANT: UD-Q4_K_XL + GGUF_FILE: gemma-4-E2B-it-UD-Q4_K_XL.gguf + MMPROJ_FILE: mmproj-F16.gguf + STUDIO_PORT: '18899' + HF_HOME: ${{ github.workspace }}/hf-cache + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: studio/frontend/package-lock.json + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + + - name: Cache HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) + id: cache-hf + uses: actions/cache@v4 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-${{ env.MMPROJ_FILE }}-v1 + + - name: Prime HF_HOME with the GGUF + mmproj + if: steps.cache-hf.outputs.cache-hit != 'true' + run: | + python -m pip install --upgrade huggingface_hub hf_transfer + mkdir -p hf-cache + HF_HUB_ENABLE_HF_TRANSFER=1 \ + hf download "$GGUF_REPO" "$GGUF_FILE" + HF_HUB_ENABLE_HF_TRANSFER=1 \ + hf download "$GGUF_REPO" "$MMPROJ_FILE" + + - name: Install Studio (--local, --no-torch) + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + & ./install.ps1 --local --no-torch 2>&1 | Tee-Object -FilePath logs/install.log + + - name: Assert install.ps1 used the Windows llama.cpp prebuilt + run: | + if grep -q "falling back to source build" logs/install.log; then + echo "::error::install.ps1 fell back to source-build llama.cpp on Windows. Studio must install the prebuilt llama-bNNNN-bin-win-cpu-x64 on Windows." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + + - name: Install OpenAI + Anthropic Python SDKs + run: python -m pip install 'openai>=1.50' 'anthropic>=0.40' + + - name: Reset auth + boot Studio (API-only) + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ + > logs/studio.log 2>&1 & + echo "STUDIO_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health, log in, change password, load model + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then + jq -e '.status == "healthy"' /tmp/health.json && break + fi + sleep 1 + done + jq -e '.status == "healthy"' /tmp/health.json + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="CIJson-$(python -c 'import secrets; print(secrets.token_urlsafe(12))')" + echo "::add-mask::$OLD" + echo "::add-mask::$NEW" + OLD_TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$OLD\"}" | jq -r .access_token) + curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/change-password" \ + -H "Authorization: Bearer $OLD_TOKEN" -H 'content-type: application/json' \ + -d "{\"current_password\":\"$OLD\",\"new_password\":\"$NEW\"}" > /dev/null + TOKEN=$(curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/auth/login" \ + -H 'content-type: application/json' \ + -d "{\"username\":\"unsloth\",\"password\":\"$NEW\"}" | jq -r .access_token) + echo "API_KEY=$TOKEN" >> "$GITHUB_ENV" + curl -fs -X POST "http://127.0.0.1:${STUDIO_PORT}/api/inference/load" \ + -H "Authorization: Bearer $TOKEN" -H 'content-type: application/json' \ + --max-time 900 \ + -d "{\"model_path\":\"$GGUF_REPO\",\"gguf_variant\":\"$GGUF_VARIANT\",\"is_lora\":false,\"max_seq_length\":2048}" \ + | jq '{status, display_name, is_vision}' + + - name: JSON schema decoding + image input + env: + BASE_URL: http://127.0.0.1:18899 + run: | + python - <<'PY' + import base64 + import json + import os + import urllib.request + from openai import OpenAI + from anthropic import Anthropic + + BASE = os.environ["BASE_URL"] + KEY = os.environ["API_KEY"] + SEED = 3407 + TEMP = 0.2 + + def post(path, body, *, timeout = 240): + req = urllib.request.Request( + f"{BASE}{path}", + data = json.dumps(body).encode(), + method = "POST", + headers = { + "Authorization": f"Bearer {KEY}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(req, timeout = timeout) as resp: + return resp.status, json.loads(resp.read().decode()) + + # ── 1. response_format = json_object (JSON mode) ───────────── + status, data = post("/v1/chat/completions", { + "model": "default", + "messages": [ + {"role": "system", "content": 'Reply with a single JSON object of the form {"city": "...", "country": "..."}. Output ONLY the JSON, nothing else.'}, + {"role": "user", "content": "What is the capital of France?"}, + ], + "temperature": TEMP, + "max_tokens": 600, + "seed": SEED, + "stream": False, + "enable_thinking": False, + "response_format": {"type": "json_object"}, + }, timeout = 600) + assert status == 200, f"json status {status}: {data}" + assert ( + isinstance(data.get("choices"), list) + and data["choices"] + and "message" in data["choices"][0] + ), f"json response envelope malformed: {data}" + content = (data["choices"][0]["message"].get("content") or "").strip() + print(f"[json] raw json_object content: {content!r}") + if content.startswith("```"): + content = content.split("```", 2)[1] + if content.startswith("json"): + content = content[4:] + content = content.strip("`\n ") + if content: + try: + parsed = json.loads(content) + if "paris" in str(parsed.get("city", "")).lower(): + print(f"[json] PASS json_object -> {parsed}") + else: + print(f"[json] WARN json_object decoded but city!=Paris: {parsed}") + except json.JSONDecodeError as exc: + print(f"[json] WARN json_object content not parseable ({exc}); content={content!r}") + else: + print("[json] WARN json_object produced empty content") + + status2, data2 = post("/v1/chat/completions", { + "model": "default", + "messages": [{"role": "user", "content": "What is the capital of France? Answer with one word."}], + "temperature": TEMP, + "max_tokens": 400, + "seed": SEED, + "stream": False, + "enable_thinking": False, + }, timeout = 600) + assert status2 == 200, f"plain status {status2}: {data2}" + plain = (data2["choices"][0]["message"].get("content") or "").lower() + print(f"[json] plain capital-of-france reply: {plain!r}") + if "paris" in plain: + print("[json] PASS plain inference path (paris mentioned)") + else: + print( + f"[json] WARN plain inference returned no 'paris' -- " + f"model output drift. HTTP path validated separately above." + ) + + # ── 2. OpenAI image_url (data URI base64) ─────────────────── + PNG_64X64_RED_B64 = ( + "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAYklEQVR4nO3PMQ0AIADAMEAI/k" + "UhBhEcDcmqYJtn7/GzpQNeNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA" + "1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaA1oDWgNaBdCJ0BmMJ25zMAAAAASUVORK5CYII=" + ) + data_uri = f"data:image/png;base64,{PNG_64X64_RED_B64}" + + # On Windows + the gemma-4-E2B mmproj, llama.cpp's vision + # path runs on CPU (no Metal involvement). The wrapper is + # kept for resilience but the vision path is expected to + # work on Windows; an exception here is a real regression. + client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY) + try: + openai_resp = client.chat.completions.create( + model = "default", + temperature = TEMP, + max_tokens = 80, + seed = SEED, + messages = [{ + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": data_uri}}, + {"type": "text", "text": "What colour dominates this image? Reply in one word."}, + ], + }], + ) + openai_text = (openai_resp.choices[0].message.content or "").lower() + print(f"[image/openai] reply: {openai_text!r}") + if openai_text: + print("[image/openai] PASS image_url accepted, non-empty response") + else: + print("[image/openai] WARN image_url accepted but empty content") + except Exception as exc: + print( + f"[image/openai] WARN image_url SDK call raised: {type(exc).__name__}: " + f"{exc}. Studio successfully forwarded the request; failure here is " + f"upstream llama.cpp vision behaviour." + ) + + # ── 3. Anthropic source/base64 image ──────────────────────── + anthropic = Anthropic( + base_url = BASE, + api_key = "unused", + default_headers = {"Authorization": f"Bearer {KEY}"}, + ) + try: + a_msg = anthropic.messages.create( + model = "default", + max_tokens = 80, + temperature = TEMP, + extra_body = {"seed": SEED}, + messages = [{ + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": PNG_64X64_RED_B64, + }, + }, + {"type": "text", "text": "Describe this image briefly."}, + ], + }], + ) + a_text = "".join(b.text for b in a_msg.content if getattr(b, "type", None) == "text") + print(f"[image/anthropic] reply: {a_text!r}") + if a_text: + print("[image/anthropic] PASS source/base64 accepted, non-empty response") + else: + print("[image/anthropic] WARN source/base64 accepted but empty content") + except Exception as exc: + print( + f"[image/anthropic] WARN anthropic image SDK call raised: " + f"{type(exc).__name__}: {exc}. Likely upstream llama.cpp vision " + f"behaviour, NOT a Studio regression." + ) + PY + + - name: Stop Studio + if: always() + run: | + kill "${STUDIO_PID}" 2>/dev/null || true + sleep 2 + + - name: Upload logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: windows-json-images-log + path: | + logs/studio.log + logs/install.log + retention-days: 7 diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml new file mode 100644 index 0000000000..a48222bac5 --- /dev/null +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Windows counterpart to studio-ui-smoke.yml / studio-mac-ui-smoke.yml. +# Same Playwright + Chromium end-to-end chat UI flow + extra UI flow, +# but on the FREE windows-latest runner so we catch Windows-specific +# regressions in the install path (install.ps1), the Studio CLI's +# Windows process-management branches, and the llama.cpp prebuilt's +# Windows HTTP layer. + +name: Windows Studio UI CI + +on: + pull_request: + paths: + - 'studio/**' + - 'unsloth/**' + - 'unsloth_cli/**' + - 'install.ps1' + - 'pyproject.toml' + - 'tests/studio/**' + - '.github/workflows/studio-windows-ui-smoke.yml' + push: + branches: [main, pip] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + ui-smoke: + name: Chat UI Tests + runs-on: windows-latest + timeout-minutes: 45 + # Default every step's shell to Git Bash. windows-latest's default + # shell is pwsh; without this each curl / heredoc / `kill $PID` + # step would need its own `shell: bash`. Steps that genuinely + # need PowerShell (install.ps1 invocation) override per-step. + defaults: + run: + shell: bash + env: + GGUF_REPO: unsloth/gemma-3-270m-it-GGUF + GGUF_VARIANT: UD-Q4_K_XL + GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18896' + HF_HOME: ${{ github.workspace }}/hf-cache + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: studio/frontend/package-lock.json + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + # No `cache: 'pip'`. install.ps1 / setup.ps1 use uv and + # never populate ~/.cache/pip; setup-python's post-step + # then fatal-errors with "Cache folder path is retrieved + # for pip but doesn't exist on disk". + + - name: Cache HF_HOME for ${{ env.GGUF_REPO }} + id: cache-hf + uses: actions/cache@v4 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v1 + + - name: Prime HF_HOME with the GGUF + if: steps.cache-hf.outputs.cache-hit != 'true' + run: | + python -m pip install --upgrade huggingface_hub hf_transfer + mkdir -p hf-cache + HF_HUB_ENABLE_HF_TRANSFER=1 \ + hf download "$GGUF_REPO" "$GGUF_FILE" + + - name: Install Studio (--local, --no-torch) + # install.ps1 is the supported Windows installer. install.sh + # has no Windows branch (apt-get / brew calls). The PS1 + # script's `Install-UnslothStudio @args` line at the bottom + # forwards `--local --no-torch` correctly. + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + & ./install.ps1 --local --no-torch 2>&1 | Tee-Object -FilePath logs/install.log + + - name: Assert install.ps1 used the Windows llama.cpp prebuilt + run: | + # Windows install must take the prebuilt path. The CPU + # asset name is llama-bNNNN-bin-win-cpu-x64.zip on + # windows-latest (no GPU). Source-build fallback here + # is a Studio bug -- we ship Windows prebuilts. + if grep -q "falling back to source build" logs/install.log; then + echo "::error::install.ps1 fell back to source-build llama.cpp on Windows. Studio must install the prebuilt llama-bNNNN-bin-win-cpu-x64 on Windows." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + if ! grep -qE "prebuilt installed and validated|prebuilt up to date and validated|bin-win-cpu-x64|bin-win-cuda-" logs/install.log; then + echo "::error::no Windows prebuilt llama.cpp marker in install.log." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + echo "install.ps1 installed the Windows prebuilt llama.cpp" + + - name: Install Playwright + Chromium + # No --with-deps on Windows: that flag installs Linux apt + # packages. windows-latest ships the system frameworks + # Chromium needs (Edge / WebView2) already. + run: | + python -m pip install 'playwright>=1.45' + python -m playwright install chromium + + - name: Reset auth + boot Studio + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p "$STUDIO_PORT" \ + > logs/studio.log 2>&1 & + echo "STUDIO_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:${STUDIO_PORT}/api/health" > /tmp/health.json; then + jq -e '.status == "healthy"' /tmp/health.json && break + fi + sleep 1 + done + jq -e '.status == "healthy"' /tmp/health.json + + - name: Pass bootstrap password to the Playwright step + run: | + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + NEW2="CIUi-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + echo "::add-mask::$OLD" + echo "::add-mask::$NEW" + echo "::add-mask::$NEW2" + echo "STUDIO_OLD_PW=$OLD" >> "$GITHUB_ENV" + echo "STUDIO_NEW_PW=$NEW" >> "$GITHUB_ENV" + echo "STUDIO_NEW2_PW=$NEW2" >> "$GITHUB_ENV" + + - name: Drive the chat UI with Playwright + env: + BASE_URL: http://127.0.0.1:18896 + PW_ART_DIR: logs/playwright + STUDIO_UI_STRICT: '1' + # windows-latest free runner is 4 vCPU / 16 GB; gemma-3- + # 270m turn latency under llama-server's CPU backend can + # crowd the 180s default (slower than ubuntu-latest on + # the same model). Keep the same generous budget the Mac + # job uses. + STUDIO_UI_TURN_TIMEOUT_MS: '540000' + run: | + mkdir -p logs/playwright + python tests/studio/playwright_chat_ui.py + + - name: Stop Studio (chat-ui ends with Shutdown click; this is belt-and-suspenders) + if: always() + run: | + kill "${STUDIO_PID}" 2>/dev/null || true + sleep 2 + + - name: Reset auth + boot Studio for extra UI tests (port 18897) + run: | + unsloth studio reset-password + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18897 \ + > logs/studio_extra.log 2>&1 & + echo "STUDIO_EXTRA_PID=$!" >> "$GITHUB_ENV" + + - name: Wait for /api/health on 18897 + run: | + for i in $(seq 1 180); do + if curl -fs "http://127.0.0.1:18897/api/health" > /tmp/health2.json; then + jq -e '.status == "healthy"' /tmp/health2.json && break + fi + sleep 1 + done + jq -e '.status == "healthy"' /tmp/health2.json + + - name: Pass bootstrap pw for extra UI test + run: | + OLD=$(cat ~/.unsloth/studio/auth/.bootstrap_password) + NEW="CIUiExtra-$(python -c 'import secrets; print(secrets.token_urlsafe(16))')" + echo "::add-mask::$OLD" + echo "::add-mask::$NEW" + echo "STUDIO_EXTRA_OLD_PW=$OLD" >> "$GITHUB_ENV" + echo "STUDIO_EXTRA_NEW_PW=$NEW" >> "$GITHUB_ENV" + + - name: Drive Compare/Recipes/Export/Studio/Settings with Playwright + env: + BASE_URL: http://127.0.0.1:18897 + STUDIO_OLD_PW: ${{ env.STUDIO_EXTRA_OLD_PW }} + STUDIO_NEW_PW: ${{ env.STUDIO_EXTRA_NEW_PW }} + PW_ART_DIR: logs/playwright_extra + STUDIO_UI_STRICT: '1' + STUDIO_UI_TURN_TIMEOUT_MS: '540000' + GGUF_REPO: ${{ env.GGUF_REPO }} + GGUF_VARIANT: ${{ env.GGUF_VARIANT }} + run: | + mkdir -p logs/playwright_extra + python tests/studio/playwright_extra_ui.py + + - name: Stop second Studio + if: always() + run: | + kill "${STUDIO_EXTRA_PID}" 2>/dev/null || true + sleep 2 + + - name: Upload Playwright artifacts + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: windows-studio-ui-smoke-artifacts + path: | + logs/studio.log + logs/studio_extra.log + logs/install.log + logs/playwright + logs/playwright_extra + retention-days: 7 diff --git a/.github/workflows/studio-windows-update-smoke.yml b/.github/workflows/studio-windows-update-smoke.yml new file mode 100644 index 0000000000..408eecf8c5 --- /dev/null +++ b/.github/workflows/studio-windows-update-smoke.yml @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Windows counterpart to studio-update-smoke.yml / +# studio-mac-update-smoke.yml. Verifies that on the FREE +# windows-latest runner: +# +# 1. install.ps1 --local --no-torch installs Studio AND auto-fetches +# the prebuilt llama.cpp Windows binary (llama-bNNNN-bin-win-cpu- +# x64 from ggml-org/llama.cpp). Hitting the source-build fallback +# is treated as an Unsloth bug -- Studio must always pick the +# prebuilt on Windows. +# 2. unsloth studio update --local is idempotent. Two consecutive +# runs both report "prebuilt up to date and validated", no +# source-build fallback. The CLI's _find_setup_script picks +# setup.ps1 on Windows automatically. +# 3. The installed Studio still boots and /api/health returns +# healthy after the update path. + +name: Windows Studio Update CI + +on: + pull_request: + paths: + - 'install.ps1' + - 'studio/setup.ps1' + - 'studio/setup.bat' + - 'studio/install_python_stack.py' + - 'studio/install_llama_prebuilt.py' + - 'studio/backend/requirements/**' + - 'unsloth_cli/commands/studio.py' + - 'pyproject.toml' + - '.github/workflows/studio-windows-update-smoke.yml' + push: + branches: [main, pip] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + update-idempotency: + name: Studio Updating Tests + runs-on: windows-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: studio/frontend/package-lock.json + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + # Don't cache pip: install.ps1 + setup.ps1 go through uv + # and never populate ~/.cache/pip; setup-python's post-step + # then fatal-errors with "Cache folder path is retrieved + # for pip but doesn't exist on disk". + + - name: Install Studio (--local, --no-torch) + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + New-Item -ItemType Directory -Force -Path logs | Out-Null + & ./install.ps1 --local --no-torch 2>&1 | Tee-Object -FilePath logs/install.log + + - name: Assert install.ps1 used the Windows llama.cpp prebuilt + run: | + if grep -q "falling back to source build" logs/install.log; then + echo "::error::install.ps1 fell back to source-build llama.cpp on Windows. Studio must install the prebuilt llama-bNNNN-bin-win-cpu-x64 on Windows." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + if ! grep -qE "prebuilt installed and validated|prebuilt up to date and validated|bin-win-cpu-x64|bin-win-cuda-" logs/install.log; then + echo "::error::no Windows prebuilt llama.cpp marker in install.log." + grep -E "llama-prebuilt|llama.cpp" logs/install.log | tail -60 + exit 1 + fi + echo "install.ps1 installed the Windows prebuilt llama.cpp" + + - name: First update should be a no-op (prebuilt already validated) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o pipefail + unsloth studio update --local 2>&1 | tee logs/update.log + if grep -q "falling back to source build" logs/update.log; then + echo "::error::studio update fell back to source-build llama.cpp on Windows." + grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60 + exit 1 + fi + if ! grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update.log; then + echo "::error::no prebuilt up-to-date marker in update.log." + grep -E "llama-prebuilt|llama.cpp" logs/update.log | tail -60 + exit 1 + fi + echo "update path took the prebuilt fast path" + + - name: Second update must also be a no-op + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -o pipefail + unsloth studio update --local 2>&1 | tee logs/update2.log + grep -q "falling back to source build" logs/update2.log && { + echo "::error::second update fell back to source build on Windows" + tail -60 logs/update2.log; exit 1; } || true + grep -qE "prebuilt up to date and validated|prebuilt installed and validated" logs/update2.log + echo "second update was clean" + + - name: Boot Studio briefly to confirm the install is still usable + run: | + mkdir -p logs + UNSLOTH_API_ONLY=1 unsloth studio -H 127.0.0.1 -p 18891 \ + > logs/studio.log 2>&1 & + PID=$! + HEALTHY="" + for i in $(seq 1 60); do + if curl -fs http://127.0.0.1:18891/api/health > /tmp/health.json; then + if python -c "import json,sys; d=json.load(open('/tmp/health.json')); sys.exit(0 if d.get('status')=='healthy' else 1)"; then + HEALTHY=1 + break + fi + fi + sleep 1 + done + if [ -z "$HEALTHY" ]; then + echo "Studio failed to come up after \`update\`" + tail -200 logs/studio.log + kill "$PID" 2>/dev/null || true + exit 1 + fi + kill "$PID" 2>/dev/null || true + echo "post-update Studio /api/health OK" + + - name: Upload update logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: windows-studio-update-log + path: | + logs/install.log + logs/update.log + logs/update2.log + logs/studio.log + retention-days: 7 diff --git a/tests/studio/playwright_chat_ui.py b/tests/studio/playwright_chat_ui.py index 66a4949f0d..e0316313bd 100644 --- a/tests/studio/playwright_chat_ui.py +++ b/tests/studio/playwright_chat_ui.py @@ -185,6 +185,21 @@ with sync_playwright() as p: lambda m: console_errors.append(m.text) if m.type == "error" else None, ) + # Per-turn HTTP-status capture: if a /v1/chat/completions request + # 4xx-rejects mid-test the symptom is a hung wait_for_function and + # a "FAIL: 1 non-benign pageerror events" line; this listener + # surfaces the underlying status codes so a flake is debuggable + # straight from the CI log without artifact spelunking. + chat_completions_responses: list[tuple[int, str]] = [] + page.on( + "response", + lambda r: ( + chat_completions_responses.append((r.status, r.url)) + if "/v1/chat/completions" in r.url + else None + ), + ) + def shoot(name): _n[0] += 1 page.screenshot( @@ -402,53 +417,83 @@ with sync_playwright() as p: "What is 2+2? Reply with the digit only.", ] - def _nonempty_bubble_count(): + def _bubble_count(): + """Total number of [data-role='assistant'] elements (empty or not).""" return page.evaluate("""() => { - const els = document.querySelectorAll('[data-role="assistant"]'); - let n = 0; - for (const el of els) { - if ((el.innerText || '').trim().length > 0) n++; - } - return n; + return document.querySelectorAll('[data-role="assistant"]').length; }""") def send_and_wait(prompt, idx): - # Wait until the previous turn's generation has fully stopped. - # If the Stop button is still attached, the prior turn is still - # streaming and we must NOT fire the next request -- doing so - # races against the assistant-ui composer's send-while-running - # gate and produces malformed messages on the wire (422 against - # /v1/chat/completions). 0 detach == the button never appeared, - # which is fine; we accept it. + # 1. Wait until the previous turn has fully stopped: Send + # button is attached AND Stop button is detached. The + # assistant-ui composer hot-swaps these inside a single + # DOM slot; relying on Stop's detached state alone is + # racy (the slot can briefly show neither during + # transition). page.wait_for_selector( 'button[aria-label="Send message"]', state = "attached", timeout = TURN_TIMEOUT_MS, ) - baseline = _nonempty_bubble_count() + try: + page.wait_for_selector( + 'button[aria-label="Stop generating"]', + state = "detached", + timeout = 5_000, + ) + except Exception: + # Stop button still hanging on -- that's the prior turn + # mid-stream. Wait it out at the full per-turn budget. + page.wait_for_selector( + 'button[aria-label="Stop generating"]', + state = "detached", + timeout = TURN_TIMEOUT_MS, + ) + + # 2. Snapshot total bubble count BEFORE send. We then wait + # for total count to grow by exactly 1 (proves the new + # placeholder rendered) and for the Stop button to come + # + go (proves the new turn ran end-to-end). We do NOT + # require the new bubble's text to be non-empty: an + # empty assistant response is a legitimate model output, + # not a test failure. The earlier "non-empty count >= + # baseline + 1" predicate broke when any prior turn + # streamed empty (which gemma-3-270m DOES on simple + # prompts at temperature 0), because that empty bubble + # became permanently "stuck" below the moving threshold. + bubbles_before = _bubble_count() composer.click() composer.fill(prompt) page.locator('button[aria-label="Send message"]').click() - # Wait for ONE more non-empty bubble than the baseline. This - # is more robust than asserting an absolute count of `idx`, - # which mis-counts if the chat already had assistant text - # before this turn (e.g. a "model loaded" hint or a streaming - # placeholder mid-generation from the prior turn). + + # 3. Wait for the new placeholder bubble to render. This + # confirms the click was actionable AND the request + # issued (assistant-ui only mounts the placeholder once + # the runtime accepts the message). page.wait_for_function( - """(target) => { - const els = document.querySelectorAll('[data-role="assistant"]'); - let n = 0; - for (const el of els) { - if ((el.innerText || '').trim().length > 0) n++; - } - return n >= target; + """(want) => { + return document.querySelectorAll( + '[data-role="assistant"]' + ).length >= want; }""", - arg = baseline + 1, + arg = bubbles_before + 1, timeout = TURN_TIMEOUT_MS, ) - # Hard-wait for streaming to end so the NEXT turn's send isn't - # racing the prior turn's tail. Take a screenshot if it doesn't - # detach in time so we can debug post-mortem. + + # 4. Wait for streaming to FINISH for this specific turn. + # We wait for Stop button to APPEAR (proves streaming + # started) with a short budget; if it never appears, + # that's fine -- gemma-3-270m can finish before the + # Stop button paints. Either way we then wait for it + # to be detached at the full per-turn budget. + try: + page.wait_for_selector( + 'button[aria-label="Stop generating"]', + state = "attached", + timeout = 3_000, + ) + except Exception: + pass try: page.wait_for_selector( 'button[aria-label="Stop generating"]', @@ -469,6 +514,17 @@ with sync_playwright() as p: if len(texts) < len(prompts): fail(f"expected >= {len(prompts)} assistant bubbles, got {len(texts)}") info(f"five turn lengths = {[len(t) for t in texts[:5]]}") + # Surface /v1/chat/completions HTTP status distribution so a flake + # is debuggable from the CI log directly. A 4xx during a chat + # turn is almost always the upstream cause of a hung + # wait_for_function on a downstream turn. + if chat_completions_responses: + statuses = [code for code, _ in chat_completions_responses] + bad = [code for code in statuses if code >= 400] + info( + f"/v1/chat/completions: {len(statuses)} request(s); " + f"statuses={statuses}; 4xx/5xx={len(bad)}" + ) # ───────────────────────────────────────────────────── # 5. Regenerate the last assistant turn.