diff --git a/.github/scripts/agent-guides-drive.sh b/.github/scripts/agent-guides-drive.sh new file mode 100755 index 0000000000..3c7cea919c --- /dev/null +++ b/.github/scripts/agent-guides-drive.sh @@ -0,0 +1,502 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Drive one coding agent against the running `unsloth run` server for the +# Local Agent Guides CI. All failures from here are failure class (c) +# "guide drift": the server preflight already passed and the agent CLI +# already installed, so a failure here means the documented recipe in +# unsloth_cli/commands/connect.py no longer produces a working flow. +# +# Self-updating: for the 5 agents with a connect.py recipe we obtain the +# exact env + command from `unsloth connect --no-launch` and run +# THAT, so a recipe change is exercised automatically. Pi (no connect.py +# command at HEAD) is driven by a hand-written recipe. +# +# Every agent invocation is wrapped in `timeout` so a headless-TTY prompt +# can never hang the runner -- a timeout is reported as guide drift with a +# distinct message. +# +# Usage: +# agent-guides-drive.sh connection +# agent-guides-drive.sh file-edit +# agent-guides-drive.sh attribution-ab claude +# +# Required env (exported by serve-unsloth-run.sh): +# UNSLOTH_BASE_URL UNSLOTH_API_KEY UNSLOTH_MODEL_ID +# UNSLOTH_LLAMA_LOG_DIR AGENT_INVOKE_TIMEOUT UNSLOTH_SEED +set -uo pipefail + +MODE="${1:?usage: agent-guides-drive.sh }" +AGENT="${2:?usage: agent-guides-drive.sh }" + +: "${UNSLOTH_BASE_URL:?serve step did not export UNSLOTH_BASE_URL}" +: "${UNSLOTH_API_KEY:?serve step did not export UNSLOTH_API_KEY}" +: "${UNSLOTH_MODEL_ID:?serve step did not export UNSLOTH_MODEL_ID}" +# Determinism (seed/temp) is applied at the server level by +# serve-unsloth-run.sh --extra; agents inherit it through the API. +TIMEOUT="${AGENT_INVOKE_TIMEOUT:-180}" + +# Claude refuses --dangerously-skip-permissions outside a sandbox; the CI runner +# IS the sandbox, so declare it (mirrors unslothai/scripts launcher.sh). Harmless +# to the other agents, which ignore it. +export IS_SANDBOX=1 + +# Absolute paths anchored at the repo root (this script lives in +# .github/scripts/). Everything writes here regardless of the current working +# directory, so the file-edit mode can `cd` into a scratch work dir without +# breaking log/redaction writes. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +LOGS_DIR="$REPO_ROOT/logs" +REDACTED_DIR="$REPO_ROOT/redacted-configs" +WORKDIR_BASE="$REPO_ROOT/agent-workdir" +CACHE_HELPER="$SCRIPT_DIR/assert-prompt-cache.sh" +mkdir -p "$LOGS_DIR" "$REDACTED_DIR" +CONNECT_REF="unsloth_cli/commands/connect.py" + +# Prefill-shrinking flags for Claude Code. The heavyweight agents send +# multi-thousand-token system prompts + full tool schemas, which on a CPU-only +# runner is minutes of prefill per model round-trip (~16 tok/s for a 4B model). +# Replacing the ~5.7k default system prompt with a tiny one (--system-prompt-file) +# and restricting tools cuts the prefill to a few hundred tokens so it completes +# quickly on CPU. These only shape the request size; the connect.py recipe +# (endpoint, auth, model) is still exercised end to end. +# +# The bulk of Claude Code's prompt is the built-in tool JSON schemas: measured +# via `claude -p /context`, the default prompt is ~28k tokens of which ~18k is +# "System tools" alone. --allowedTools/--disallowedTools only gate PERMISSION to +# call a tool; they do NOT remove its schema from what is sent to the model, so +# the earlier whitelist left the full ~18k in the prompt and CPU prefill +# (~16 tok/s) overran claude's own request timeout into a retry loop. --tools is +# the flag that restricts which schemas are sent. (The ~8k "Memory files" chunk +# is auto-loaded CLAUDE.md; the unsloth repo ships none, so it is 0 in CI.) +# +# Connection probe: --tools "" sends ZERO tool schemas, leaving ~20 tokens total +# (a one-line --system-prompt-file + the user turn), which prefills instantly. +CLAUDE_CONNECT_FLAGS=( + --system-prompt-file "$SCRIPT_DIR/ci-connect-prompt.txt" + --tools "" +) +# File-edit: the task needs the file/shell tools, so send only those schemas +# (~2.3k tokens vs ~18k for the full set). +CLAUDE_EDIT_FLAGS=( + --system-prompt-file "$SCRIPT_DIR/ci-min-system-prompt.txt" + --tools "Bash,Edit,Write,Read" +) + +guide_fail() { + echo "::error::[guide drift] agent=${AGENT}: $* (preflight passed + install OK, so the documented flow in ${CONNECT_REF} drifted)." >&2 + exit 1 +} + +# Redact the API key from any file we are about to keep as an artifact. +# Portable across GNU sed (Linux runners) and BSD sed (macOS), so the +# redaction is never silently skipped. +redact() { + local f + for f in "$@"; do + [ -f "$f" ] || continue + if sed --version >/dev/null 2>&1; then + sed -i "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + else + sed -i '' "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + fi + done +} + +# A reply must be non-empty and free of connection/auth errors. +assert_reply() { + local out="$1" + if [ ! -s "$out" ]; then + guide_fail "agent produced an EMPTY reply" + fi + if grep -qiE 'connection refused|connection error|econnrefused|fetch failed|http 4[0-9][0-9]|unauthorized|invalid api key|authentication failed' "$out"; then + guide_fail "agent reply contained a connection/auth error: $(grep -iE 'connection|unauthorized|auth|http 4' "$out" | head -1)" + fi + echo "[$AGENT] reply (first 20 lines):" + head -20 "$out" +} + +# Run a command under a hard timeout; map 124 to a guide-drift hang message. +run_timed() { # $1=outfile, rest=command + local out="$1"; shift + timeout "$TIMEOUT" "$@" > "$out" 2>&1 + local rc=$? + if [ "$rc" -eq 124 ]; then + redact "$out" # guide_fail exits below, so scrub the transcript here too + echo "[$AGENT] last 40 lines before timeout:"; tail -40 "$out" 2>/dev/null || true + guide_fail "invoke timed out after ${TIMEOUT}s (headless-TTY hang -- the recipe likely needs a non-interactive/print flag)" + fi + return "$rc" +} + +# ── Pi: no connect.py command at HEAD -> hand-written recipe ────────────── +write_pi_config() { + if unsloth connect pi --help >/dev/null 2>&1; then + # Tripwire: once a real recipe exists, the hand-written config would mask any + # drift in it, defeating the point of this CI. Fail hard so the cell is + # migrated to the self-updating `unsloth connect pi --no-launch` path. + guide_fail "connect.py now ships a 'pi' command -- migrate this CI cell to the 'unsloth connect pi --no-launch' path so the documented recipe is exercised (the hand-written Pi config no longer reflects it)" + fi + mkdir -p "$HOME/.pi/agent" + python3 - "$UNSLOTH_BASE_URL" "$UNSLOTH_API_KEY" "$UNSLOTH_MODEL_ID" <<'PY' +import json, os, sys +base, key, model = sys.argv[1], sys.argv[2], sys.argv[3] +cfg = {"providers": {"unsloth": { + "api": "openai-completions", + "baseUrl": f"{base}/v1", + "apiKey": key, + "models": [{"id": model}], +}}} +path = os.path.expanduser("~/.pi/agent/models.json") +with open(path, "w") as fh: + json.dump(cfg, fh, indent=2) +PY + cp "$HOME/.pi/agent/models.json" "$REDACTED_DIR/pi-models.json" 2>/dev/null || true + redact "$REDACTED_DIR/pi-models.json" +} + +# ── 5-agent connect.py path: parse env + command from --no-launch ───────── +# Populates globals CONNECT_ENV (export/unset lines) and CONNECT_CMD (the +# launch command on the last printed line), and runs connect.py's config +# writers as a side effect (it writes ~/.codex, ~/.claude, etc.). +parse_connect() { + local raw="$LOGS_DIR/connect-${AGENT}.txt" + if ! unsloth connect "$AGENT" --no-launch --api-key "$UNSLOTH_API_KEY" > "$raw" 2>&1; then + cat "$raw" + guide_fail "'unsloth connect ${AGENT} --no-launch' exited non-zero" + fi + echo "[$AGENT] connect --no-launch printed:"; cat "$raw" + CONNECT_ENV="$(grep -E '^(export |unset )' "$raw" || true)" + # The launch command is the last non-export, non-status line. connect.py + # prints "Studio · model " and "Updated ..." status lines first. + CONNECT_CMD="$(grep -vE '^(export |unset |Studio |Updated |Disabled |Warning|Loading)' "$raw" \ + | grep -E '[^[:space:]]' | tail -1)" + [ -n "$CONNECT_CMD" ] || guide_fail "could not parse a launch command from connect --no-launch output" + redact "$raw" +} + +# Cross-check the documented contract knobs so silent connect.py changes +# (env-var rename, wire_api flip, attribution setting drop) also fail/flag. +crosscheck_contract() { + local raw="$LOGS_DIR/connect-${AGENT}.txt" + case "$AGENT" in + codex) + grep -q 'UNSLOTH_STUDIO_AUTH_TOKEN' "$raw" \ + || guide_fail "Codex env key is no longer UNSLOTH_STUDIO_AUTH_TOKEN (connect.py _CODEX_ENV_KEY)" + if [ -f "$HOME/.codex/config.toml" ]; then + grep -q 'wire_api = "responses"' "$HOME/.codex/config.toml" \ + || guide_fail "Codex wire_api is no longer \"responses\" in ~/.codex/config.toml" + cp "$HOME/.codex/config.toml" "$REDACTED_DIR/codex-config.toml" + fi + grep -q 'codex --oss --profile unsloth_api' "$raw" \ + || echo "::warning::Codex launch command changed from 'codex --oss --profile unsloth_api'" + ;; + claude) + grep -q 'ANTHROPIC_AUTH_TOKEN' "$raw" \ + || guide_fail "Claude no longer exports ANTHROPIC_AUTH_TOKEN (connect.py claude())" + if [ -f "$HOME/.claude/settings.json" ]; then + grep -q '"CLAUDE_CODE_ATTRIBUTION_HEADER"' "$HOME/.claude/settings.json" \ + || echo "::warning::CLAUDE_CODE_ATTRIBUTION_HEADER not written to ~/.claude/settings.json (ensure_claude_attribution_header)" + cp "$HOME/.claude/settings.json" "$REDACTED_DIR/claude-settings.json" + fi + ;; + hermes) + grep -q 'UNSLOTH_API_KEY' "$raw" \ + || guide_fail "Hermes env key is no longer UNSLOTH_API_KEY (connect.py _HERMES_ENV_KEY)" + [ -f "$HOME/.hermes/config.yaml" ] && cp "$HOME/.hermes/config.yaml" "$REDACTED_DIR/hermes-config.yaml" + ;; + openclaw) + if [ -f "$HOME/.openclaw/openclaw.json" ]; then + grep -q '"openai-completions"' "$HOME/.openclaw/openclaw.json" \ + || echo "::warning::OpenClaw provider api is no longer 'openai-completions' (write_openclaw_config)" + cp "$HOME/.openclaw/openclaw.json" "$REDACTED_DIR/openclaw.json" + fi + ;; + opencode) + [ -f "$HOME/.config/opencode/opencode.json" ] && cp "$HOME/.config/opencode/opencode.json" "$REDACTED_DIR/opencode.json" + ;; + esac + redact "$REDACTED_DIR"/* 2>/dev/null || true +} + +# Heavyweight agents (hermes, openclaw) bake a large system prompt + tool JSON +# schemas into every request, which a CPU runner cannot prefill before the invoke +# timeout. As with claude's --tools, we shrink the request from the agent's own +# config: zero tools for the connection probe collapses the prompt to a few +# hundred tokens, since both CLIs gate the bulk of their prompt on having tools. + +# Hermes: an explicit empty cli toolset disables all tools (and drops the +# tool-gated guidance blocks), so -z sends ~300 tokens instead of thousands. +# hermes ships a DEFAULT config.yaml that already has a populated +# platform_toolsets, and `unsloth connect` merges into it, so we must override +# cli (not just append). That needs a YAML parser, and the runner's bare +# python3 has no PyYAML -- but the venv that ships `unsloth` does (connect.py +# imports yaml), so run the patch with that interpreter. +# (-z reads platform_toolsets.cli; --ignore-rules is a no-op under -z.) +patch_hermes_tools() { # $1 = none|default + # Find a python that can import yaml. The runner's bare python3 cannot, but the + # interpreter in the `unsloth` console-script shebang provably can (it runs + # connect.py's write_hermes_config, which imports yaml). Try that first, then + # any python on PATH, then the venv sibling, picking the first with PyYAML. + local cand py="" shebang + shebang="$(head -1 "$(command -v unsloth)" 2>/dev/null | sed -n 's/^#![[:space:]]*//p' | awk '{print $1}')" + for cand in "$shebang" python3 python "$(dirname "$(command -v unsloth)")/python"; do + [ -n "$cand" ] || continue + { [ -x "$cand" ] || command -v "$cand" >/dev/null 2>&1; } || continue + if "$cand" -c 'import yaml' 2>/dev/null; then py="$cand"; break; fi + done + [ -n "$py" ] || guide_fail "could not find a python with PyYAML to patch ~/.hermes/config.yaml" + echo "[hermes] patching config with $py" + "$py" - "$1" <<'PY' +import os, sys +import yaml +mode = sys.argv[1] +p = os.path.expanduser("~/.hermes/config.yaml") +cfg = (yaml.safe_load(open(p)) or {}) if os.path.exists(p) else {} +ts = cfg.get("platform_toolsets") +if not isinstance(ts, dict): + ts = cfg["platform_toolsets"] = {} +if mode == "none": + ts["cli"] = [] # explicit empty list -> zero tools (not "defaults") +else: + ts.pop("cli", None) # file-edit needs real tools -> restore defaults +with open(p, "w") as fh: + yaml.safe_dump(cfg, fh, sort_keys=False) +print(f"[hermes] platform_toolsets.cli = {ts.get('cli', 'default')}") +PY +} + +# OpenClaw: 'openclaw agent' has no tool/prompt flags, so we define a 'ci' agent +# in openclaw.json. tools.deny ["*"] sends zero tool schemas (deny always wins) +# for the connection probe; contextInjection "never" + defaults.skipBootstrap +# drop the auto-injected AGENTS.md/SOUL.md bootstrap (the bulk of the prompt) for +# both modes. --agent must reference a defined agent, so write it before invoking. +patch_openclaw_agent() { # $1 = notools|tools + python3 - "$1" <<'PY' +import os, sys, json +mode = sys.argv[1] +p = os.path.expanduser("~/.openclaw/openclaw.json") +cfg = json.load(open(p)) if os.path.exists(p) else {} +agents = cfg.setdefault("agents", {}) +agents.setdefault("defaults", {})["skipBootstrap"] = True +lst = [a for a in agents.get("list", []) if a.get("id") != "ci"] +agent = {"id": "ci", "contextInjection": "never"} +if mode == "notools": + agent["tools"] = {"deny": ["*"]} +lst.append(agent) +agents["list"] = lst +with open(p, "w") as fh: + json.dump(cfg, fh, indent=2) +print(f"[openclaw] agent ci tools = {agent.get('tools', 'default')}") +PY +} + +# Build an invoke script that applies connect.py's env then runs the launch +# command (with extra args appended) under bash. We do NOT eval connect's env +# into this shell; we write it into a one-shot script so the export/unset +# semantics are exactly what connect.py printed. The script path is absolute +# so it is valid even when the caller has cd'd into a scratch work dir. +invoke_via_connect() { # $1=outfile, rest=extra args appended to the command + local out="$1"; shift + local script="$LOGS_DIR/invoke-${AGENT}.sh" + local real; real="$(mktemp)" + { + echo "set -uo pipefail" + echo "$CONNECT_ENV" + # Append extra args (the prompt / flags) to the launch command verbatim. + printf '%s' "$CONNECT_CMD" + local a + for a in "$@"; do printf ' %q' "$a"; done + printf '\n' + } > "$real" + # Upload a REDACTED copy of the script, but EXECUTE the un-redacted one from a + # temp path outside the artifact dir. Redacting the script we run would turn + # the real `export TOKEN=sk-...` line into `export TOKEN=`, which is + # invalid bash (the `<`/`>` are redirections) and silently breaks every agent. + # Writing the redacted copy up front keeps the key out of the artifact even if + # the run times out (run_timed exits before returning here). + cp "$real" "$script"; redact "$script" + echo "[$AGENT] invoking (timeout ${TIMEOUT}s): $CONNECT_CMD $*" + run_timed "$out" bash "$real" + local rc=$? + rm -f "$real" + redact "$out" # the transcript can echo the token; scrub before upload + return "$rc" +} + +# ═════════════════════════════════════════════════════════════════════════ +case "$MODE" in + # ── connection: trivial prompt, assert a non-empty, error-free reply ──── + connection) + PROMPT='Reply with exactly the single word: pong' + OUT="$LOGS_DIR/${AGENT}-connection.txt" + if [ "$AGENT" = "pi" ]; then + write_pi_config + run_timed "$OUT" pi -p --provider unsloth --model "$UNSLOTH_MODEL_ID" "$PROMPT" + else + parse_connect + crosscheck_contract + # claude/codex run in print mode via the flags connect.py emits + # (claude -p / codex exec). For agents whose default subcommand prints + # to stdout we pass the prompt through ctx.args. + case "$AGENT" in + claude) invoke_via_connect "$OUT" "${CLAUDE_CONNECT_FLAGS[@]}" -p "$PROMPT" ;; + codex) invoke_via_connect "$OUT" exec --dangerously-bypass-approvals-and-sandbox "$PROMPT" ;; + opencode) invoke_via_connect "$OUT" run "$PROMPT" ;; + hermes) patch_hermes_tools none + invoke_via_connect "$OUT" -z "$PROMPT" ;; + openclaw) patch_openclaw_agent notools + invoke_via_connect "$OUT" agent --local --agent ci \ + --model "unsloth/${UNSLOTH_MODEL_ID}" --message "$PROMPT" ;; + *) invoke_via_connect "$OUT" "$PROMPT" ;; + esac + fi + # A non-zero exit from the documented launch command is drift even if it + # printed something: a benign-looking "command not found" / usage dump would + # otherwise slip past assert_reply (which only flags empty/error-keyword text). + rc=$? + [ "$rc" -eq 0 ] || guide_fail "the documented launch command exited non-zero (rc=$rc) -- see the transcript above" + assert_reply "$OUT" + echo "[$AGENT] connection OK" + ;; + + # ── file-edit: deterministic 2-turn hello.py test (Qwen3.5-2B) ────────── + file-edit) + WORK="$WORKDIR_BASE/${AGENT}" + rm -rf "$WORK"; mkdir -p "$WORK" + OUT1="$LOGS_DIR/${AGENT}-fileedit-turn1.txt" + OUT2="$LOGS_DIR/${AGENT}-fileedit-turn2.txt" + T1='Create a file named hello.py in the current directory whose entire contents are a single line: print("Hello"). Do not run it.' + T2='Run hello.py with python and show me the exact output.' + + # The connect.py recipe writers + crosscheck must see the repo; run them + # from the repo root BEFORE cd-ing into the scratch work dir. + if [ "$AGENT" != "pi" ]; then + parse_connect + crosscheck_contract + # File-edit needs real tools, so we cannot zero them as in connection. + # hermes keeps default tools; openclaw still strips its AGENTS.md/SOUL.md + # bootstrap (the largest prompt chunk) via the 'ci' agent. The scratch work + # dir is empty, so no project context files are auto-loaded either. + case "$AGENT" in + hermes) patch_hermes_tools default ;; + openclaw) patch_openclaw_agent tools ;; + esac + else + write_pi_config + fi + + # Drive from inside the work dir so the agent edits files there. All log + # writes use absolute $LOGS_DIR, so cwd does not matter for them. + cd "$WORK" || guide_fail "could not enter work dir $WORK" + + invoke_turn() { # $1=outfile $2=continue? $3=prompt + local out="$1" cont="$2" prompt="$3" + case "$AGENT" in + pi) run_timed "$out" pi -p --provider unsloth --model "$UNSLOTH_MODEL_ID" "$prompt" ;; + claude) + # --dangerously-skip-permissions lets headless claude actually use the + # Write/Bash tools (otherwise it blocks on an approval prompt and emits + # nothing). IS_SANDBOX=1 (exported above) authorizes it. + if [ "$cont" = "continue" ]; then + invoke_via_connect "$out" "${CLAUDE_EDIT_FLAGS[@]}" --dangerously-skip-permissions -p --continue "$prompt" + else + invoke_via_connect "$out" "${CLAUDE_EDIT_FLAGS[@]}" --dangerously-skip-permissions -p "$prompt" + fi ;; + codex) + # --dangerously-bypass-approvals-and-sandbox gives codex exec + # workspace-write (default is read-only -> cannot create hello.py) and + # skips the bubblewrap sandbox that the runner lacks. + if [ "$cont" = "continue" ]; then + invoke_via_connect "$out" exec --dangerously-bypass-approvals-and-sandbox resume --last "$prompt" + else + invoke_via_connect "$out" exec --dangerously-bypass-approvals-and-sandbox "$prompt" + fi ;; + opencode) invoke_via_connect "$out" run "$prompt" ;; + hermes) invoke_via_connect "$out" -z "$prompt" ;; + openclaw) invoke_via_connect "$out" agent --local --agent ci \ + --model "unsloth/${UNSLOTH_MODEL_ID}" --message "$prompt" ;; + *) invoke_via_connect "$out" "$prompt" ;; + esac + } + + # Turn 1: create hello.py. + invoke_turn "$OUT1" fresh "$T1" + # Fail on a non-zero agent exit before trusting side effects: an agent can + # error out (API/tool failure) yet leave a plausible file/transcript behind, + # which would otherwise slip past the assertions below (mirrors connection). + rc=$? + [ "$rc" -eq 0 ] || { echo "[$AGENT] turn-1 transcript:"; tail -40 "$OUT1" 2>/dev/null || true; \ + guide_fail "turn 1 (create hello.py) exited non-zero (rc=$rc)"; } + + # Hard assertions on the side effect (the real test): file + content + run. + if [ ! -f hello.py ]; then + echo "[$AGENT] turn-1 transcript:"; tail -40 "$OUT1" 2>/dev/null || true + guide_fail "turn 1 did not create hello.py" + fi + grep -q 'Hello' hello.py || guide_fail "hello.py does not contain 'Hello'" + RUN_OUT="$(python3 hello.py 2>&1 || true)" + [ "$RUN_OUT" = "Hello" ] || guide_fail "python3 hello.py printed '$RUN_OUT', expected exactly 'Hello'" + echo "[$AGENT] turn 1 OK (file created, prints 'Hello')" + + # Turn 2: same cwd + session continuation; assert the agent's run output + # contains Hello. Narration drift is WARN-only, missing output is a hard fail. + invoke_turn "$OUT2" continue "$T2" + rc=$? + [ "$rc" -eq 0 ] || { echo "[$AGENT] turn-2 transcript:"; tail -60 "$OUT2" 2>/dev/null || true; \ + guide_fail "turn 2 (run hello.py) exited non-zero (rc=$rc)"; } + if grep -q 'Hello' "$OUT2"; then + echo "[$AGENT] turn 2 OK (run output contains 'Hello')" + else + echo "[$AGENT] turn-2 transcript:"; tail -60 "$OUT2" 2>/dev/null || true + guide_fail "turn 2 run/bash output did not contain 'Hello'" + fi + cd "$REPO_ROOT" || true + echo "[$AGENT] file-edit OK" + ;; + + # ── attribution-ab: Claude Code KV-cache HIT vs MISS ──────────────────── + attribution-ab) + [ "$AGENT" = "claude" ] || guide_fail "attribution-ab only applies to claude" + # The llama-server log filename uses the INTERNAL random llama.cpp port, + # not STUDIO_PORT, so we never glob by port: assert-prompt-cache.sh picks + # the newest llama-*.log and we slice it by a byte offset (`mark`) captured + # right before the measured turn, so an earlier turn's reuse can't leak in. + LLAMA_LOG_DIR="${UNSLOTH_LLAMA_LOG_DIR:-$HOME/.unsloth/studio/logs/llama-server}" + export LLAMA_LOG_DIR + parse_connect # writes ~/.claude/settings.json (header=0) + env + crosscheck_contract + PROMPT='Reply with exactly the single word: pong' + + # Phase A: header DISABLED (=0, the documented setting) -> expect a HIT on + # the continued turn. connect.py's ensure_claude_attribution_header() set 0. + invoke_via_connect "$LOGS_DIR/claude-ab-hit-1.txt" -p "$PROMPT" # turn 1 primes + FROM_HIT="$(bash "$CACHE_HELPER" mark)" # offset before turn 2 + invoke_via_connect "$LOGS_DIR/claude-ab-hit-2.txt" -p --continue "$PROMPT again" + CACHE_LOG_FROM="$FROM_HIT" bash "$CACHE_HELPER" log HIT + + # Phase B: header ENABLED -> expect a MISS. The header prepends a + # per-request-changing attribution line to the system prompt, so the shared + # prefix changes every turn and the KV cache is invalidated (~90% slower); + # this is exactly what the guide flag prevents. + python3 - <<'PY' +import json, os +p = os.path.expanduser("~/.claude/settings.json") +s = json.load(open(p)) if os.path.exists(p) else {} +s.setdefault("env", {})["CLAUDE_CODE_ATTRIBUTION_HEADER"] = "1" +json.dump(s, open(p, "w"), indent=2) +PY + invoke_via_connect "$LOGS_DIR/claude-ab-miss-1.txt" -p "$PROMPT" + FROM_MISS="$(bash "$CACHE_HELPER" mark)" + invoke_via_connect "$LOGS_DIR/claude-ab-miss-2.txt" -p --continue "$PROMPT again" + CACHE_LOG_FROM="$FROM_MISS" bash "$CACHE_HELPER" log MISS + echo "[claude] attribution A/B OK (header=0 HIT, header=1 MISS)" + ;; + + *) + echo "agent-guides-drive.sh: unknown mode '$MODE'" >&2 + exit 2 + ;; +esac diff --git a/.github/scripts/agent-guides-install.sh b/.github/scripts/agent-guides-install.sh new file mode 100755 index 0000000000..dfab8aec80 --- /dev/null +++ b/.github/scripts/agent-guides-install.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Install one coding-agent CLI for the Local Agent Guides CI. Isolated as +# failure class (b) "agent package install failed": npm/curl flakiness here +# is the single biggest source of false reds, so installs retry with +# backoff and the only ::error:: this script can emit is class (b). The +# install recipes mirror the install_hint strings in +# unsloth_cli/commands/connect.py at HEAD. +# +# Usage: agent-guides-install.sh +# agent in: claude codex hermes openclaw opencode pi +set -uo pipefail + +AGENT="${1:?usage: agent-guides-install.sh }" +mkdir -p logs +LOG="logs/install-${AGENT}.log" + +install_fail() { + echo "::error::[agent install failed] agent=${AGENT}: $* (class (b): the agent CLI did not install; not a server or guide problem)." >&2 + echo "---- tail $LOG ----" >&2 + tail -60 "$LOG" 2>/dev/null || true + exit 1 +} + +# npm registry flakiness is common in CI; retry 3x with linear backoff. +npm_retry() { + local pkg="$1" i + for i in 1 2 3; do + if npm install -g "$pkg" >> "$LOG" 2>&1; then + return 0 + fi + echo "[install] npm install -g $pkg attempt $i failed; backing off $((i * 10))s" | tee -a "$LOG" + sleep "$((i * 10))" + done + return 1 +} + +# curl|bash installers, retried at the curl layer. We download to a temp file +# first and only execute on a fully successful fetch, so a truncated download +# (network hiccup mid-stream) can never run a half-written installer. +curl_bash() { + local url="$1"; shift + local i tmp + tmp="$(mktemp)" + for i in 1 2 3; do + if curl -fsSL --retry 3 --retry-delay 5 "$url" -o "$tmp" 2>>"$LOG" \ + && bash "$tmp" "$@" >> "$LOG" 2>&1; then + rm -f "$tmp" + return 0 + fi + echo "[install] curl|bash $url attempt $i failed; backing off $((i * 10))s" | tee -a "$LOG" + sleep "$((i * 10))" + done + rm -f "$tmp" + return 1 +} + +echo "[install] agent=$AGENT (log=$LOG)" +case "$AGENT" in + claude) + # connect.py install_hint: curl -fsSL https://claude.ai/install.sh | bash + curl_bash "https://claude.ai/install.sh" || install_fail "claude installer failed" + # The installer drops the binary under ~/.local/bin. + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + ;; + codex) + # connect.py install_hint: npm install -g @openai/codex + npm_retry "@openai/codex" || install_fail "npm install -g @openai/codex failed" + ;; + opencode) + # connect.py install_hint: npm install -g opencode-ai + npm_retry "opencode-ai" || install_fail "npm install -g opencode-ai failed" + ;; + openclaw) + # connect.py install_hint: curl -fsSL https://openclaw.ai/install.sh | bash + # npm is the more deterministic path in CI and matches the agent's docs; + # fall back to the connect.py curl installer if the npm tag is missing. + if ! npm_retry "openclaw@latest"; then + curl_bash "https://openclaw.ai/install.sh" || install_fail "openclaw install failed (npm + curl)" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + fi + ;; + hermes) + # connect.py install_hint: + # curl -fsSL .../NousResearch/hermes-agent/main/scripts/install.sh | bash + curl_bash "https://raw.githubusercontent.com/NousResearch/hermes-agent/main/scripts/install.sh" \ + --non-interactive --skip-setup --skip-browser --no-skills \ + || install_fail "hermes installer failed" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + ;; + pi) + # No connect.py recipe; the agent's documented package name. The CLI moved + # from the now-deprecated @mariozechner scope to @earendil-works (the old + # scope is frozen, so installing it would test a stale Pi against the API). + npm_retry "@earendil-works/pi-coding-agent" \ + || install_fail "npm install -g @earendil-works/pi-coding-agent failed" + ;; + *) + install_fail "unknown agent '$AGENT'" + ;; +esac + +echo "[install] OK for $AGENT" diff --git a/.github/scripts/assert-prompt-cache.sh b/.github/scripts/assert-prompt-cache.sh new file mode 100755 index 0000000000..f5b6b075eb --- /dev/null +++ b/.github/scripts/assert-prompt-cache.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Prompt-cache (KV-cache prefix reuse) detection, two strategies in one helper: +# +# mode=api A 2-turn /v1/chat/completions probe. Turn 2 prepends turn 1 + +# its reply, so the shared prefix must be served from llama.cpp's +# KV cache. Asserts usage.prompt_tokens_details.cached_tokens > 0 +# on turn 2. This is the OpenAI-dialect server cache sanity. +# WHY this works on chat completions: the chat path forwards +# llama-server's real cached_tokens through +# studio/backend/routes/inference.py:482-489 (_prompt_tokens_details) +# into prompt_tokens_details (inference.py:519). +# +# mode=log Read the llama-server log and decide HIT vs MISS from the +# prompt-reprocessing trace. WHY the log (not the API field): +# the Anthropic /v1/messages path builds AnthropicUsage( +# input_tokens=..., output_tokens=...) at inference.py:8787-8790 +# / :8829-8832 and NEVER sets cache_read_input_tokens, which +# therefore stays at its model default of 0 +# (studio/backend/models/inference.py:1655). So an Anthropic-path +# client (Claude Code, OpenClaw is openai-completions but Claude +# Code is the canonical Anthropic agent) can get a real KV-cache +# hit that the API usage field reports as 0. The only ground +# truth for the Anthropic path is the llama-server log. +# +# Log location (verified): studio/backend/core/inference/llama_cpp.py:4363-4365 +# _swa_cache_path().parent/"logs"/"llama-server"/llama-[label]-port-

[-try].log +# _swa_cache_path() => $UNSLOTH_STUDIO_HOME|$STUDIO_HOME or ~/.unsloth/studio +# (llama_cpp.py:337-340). So default: ~/.unsloth/studio/logs/llama-server/. +# +#

is the INTERNAL llama-server port (self._find_free_port(), +# llama_cpp.py:3489 / :4641) -- a RANDOM port, NOT the Studio port. So we must +# NOT filter the log glob by STUDIO_PORT (the brief's `port-` +# glob would never match). We pick the newest llama-*.log instead. +# +# Usage: +# assert-prompt-cache.sh api BASE_URL API_KEY +# assert-prompt-cache.sh log EXPECT # EXPECT = HIT | MISS +# # reads MARKER_BEFORE/MARKER_AFTER +# # byte offsets from env (see below) +# assert-prompt-cache.sh mark # print current log size to stdout +# # (use to bracket a turn) +# +# Env for mode=log: +# LLAMA_LOG_DIR override the log dir (default ~/.unsloth/studio/logs/llama-server) +# CACHE_LOG_FROM byte offset to start scanning the newest log from (so we +# only look at the trace produced by THIS turn). Default 0. +# +# Exit codes: 0 = assertion held; 1 = assertion failed (::error:: emitted). + +set -uo pipefail + +MODE="${1:?usage: assert-prompt-cache.sh api|log|mark ...}" + +# --------------------------------------------------------------------------- +# Locate the newest llama-server log. Shared by mark + log modes. +# --------------------------------------------------------------------------- +_default_log_dir() { + local home="${UNSLOTH_STUDIO_HOME:-${STUDIO_HOME:-}}" + if [ -n "$home" ]; then + echo "${home%/}/logs/llama-server" + else + echo "${HOME}/.unsloth/studio/logs/llama-server" + fi +} + +_newest_log() { + local dir="${LLAMA_LOG_DIR:-$(_default_log_dir)}" + [ -d "$dir" ] || return 1 + # Newest by mtime among llama-*.log (covers both `llama--port-

.log` + # and the retry form `llama-

-try.log`). Filenames are + # tool-generated timestamps, so ls -t is safe here. + # shellcheck disable=SC2012 + ls -1t "$dir"/llama-*.log 2>/dev/null | head -1 +} + +case "$MODE" in + # ------------------------------------------------------------------------- + # mark: emit the current byte size of the newest llama log so a caller can + # scan only the slice a single turn produced (set CACHE_LOG_FROM to it). + # ------------------------------------------------------------------------- + mark) + log="$(_newest_log || true)" + if [ -n "$log" ] && [ -f "$log" ]; then + wc -c < "$log" | tr -d ' ' + else + echo 0 + fi + exit 0 + ;; + + # ------------------------------------------------------------------------- + # api: 2-turn /v1/chat/completions, assert turn-2 cached_tokens > 0. + # ------------------------------------------------------------------------- + api) + BASE_URL="${2:?usage: assert-prompt-cache.sh api BASE_URL API_KEY}" + API_KEY="${3:?usage: assert-prompt-cache.sh api BASE_URL API_KEY}" + + # A deliberately long, fixed system prompt makes the shared prefix big so a + # KV-cache hit is unambiguous (cached_tokens grows with the reused prefix). + SYS='You are a meticulous assistant. Always answer concisely and correctly. This is a fixed system preamble that exists only to create a large, identical prompt prefix across both turns so the KV cache has something substantial to reuse on the second request. Do not mention this preamble.' + + turn1_body() { + jq -n --arg sys "$SYS" '{ + model: "default", + messages: [ + {role:"system", content:$sys}, + {role:"user", content:"What is the capital of France?"} + ], + temperature: 0.0, seed: 3407, max_tokens: 40, stream: false, + enable_thinking: false + }' + } + + echo "[cache/api] turn 1 (prime the KV cache)" + R1="$(curl -fs -X POST "${BASE_URL}/v1/chat/completions" \ + -H "Authorization: Bearer ${API_KEY}" -H 'content-type: application/json' \ + --max-time 240 -d "$(turn1_body)")" || { + echo "::error::[cache/api] turn-1 /v1/chat/completions request failed. Unsloth server/API regression." + exit 1 + } + A1="$(echo "$R1" | jq -r '.choices[0].message.content // ""')" + + turn2_body() { + jq -n --arg sys "$SYS" --arg a1 "$A1" '{ + model: "default", + messages: [ + {role:"system", content:$sys}, + {role:"user", content:"What is the capital of France?"}, + {role:"assistant", content:$a1}, + {role:"user", content:"And the capital of Germany?"} + ], + temperature: 0.0, seed: 3407, max_tokens: 40, stream: false, + enable_thinking: false + }' + } + + echo "[cache/api] turn 2 (expect cached_tokens > 0)" + R2="$(curl -fs -X POST "${BASE_URL}/v1/chat/completions" \ + -H "Authorization: Bearer ${API_KEY}" -H 'content-type: application/json' \ + --max-time 240 -d "$(turn2_body)")" || { + echo "::error::[cache/api] turn-2 /v1/chat/completions request failed. Unsloth server/API regression." + exit 1 + } + + CACHED="$(echo "$R2" | jq -r '.usage.prompt_tokens_details.cached_tokens // 0')" + PROMPT_TOK="$(echo "$R2" | jq -r '.usage.prompt_tokens // 0')" + echo "[cache/api] turn-2 usage: prompt_tokens=${PROMPT_TOK} cached_tokens=${CACHED}" + + if [ -z "$CACHED" ] || ! [ "$CACHED" -gt 0 ] 2>/dev/null; then + echo "::error::[cache/api] turn-2 usage.prompt_tokens_details.cached_tokens=${CACHED}, expected > 0. The server is not surfacing llama.cpp KV-cache hits on /v1/chat/completions. Check studio/backend/routes/inference.py:482-489 (_prompt_tokens_details) and :519. Full turn-2 usage:" + echo "$R2" | jq -c '.usage' 2>/dev/null || echo "$R2" + exit 1 + fi + echo "[cache/api] PASS server cache sanity (cached_tokens=${CACHED} > 0)" + exit 0 + ;; + + # ------------------------------------------------------------------------- + # log: classify the newest llama-server log (from CACHE_LOG_FROM bytes on) + # as HIT or MISS and compare to EXPECT. + # ------------------------------------------------------------------------- + log) + EXPECT="${2:?usage: assert-prompt-cache.sh log HIT|MISS}" + FROM="${CACHE_LOG_FROM:-0}" + + log="$(_newest_log || true)" + if [ -z "$log" ] || [ ! -f "$log" ]; then + echo "::error::[cache/log] no llama-server log under ${LLAMA_LOG_DIR:-$(_default_log_dir)}. Cannot read KV-cache trace. (Path contract: studio/backend/core/inference/llama_cpp.py:4363-4365.)" + exit 1 + fi + echo "[cache/log] reading $log from byte $FROM" + + # Scan only the slice produced after FROM. + slice="$(tail -c "+$((FROM + 1))" "$log" 2>/dev/null || cat "$log")" + + # ---- HIT detectors (most-specific first) ----------------------------- + # 1. Modern + legacy "re-used N tokens" / "reused N" (N>0). Primary signal + # per the design brief. + reused_n="$(printf '%s\n' "$slice" \ + | grep -aoiE 're-?used[^0-9]*([0-9]+)' \ + | grep -aoE '[0-9]+' | sort -rn | head -1 || true)" + # 2. "kv cache rm [START, end)" with START>0 => prefix [0,START) reused. + cache_rm_start="$(printf '%s\n' "$slice" \ + | grep -aoiE 'kv cache rm \[[0-9]+' \ + | grep -aoE '[0-9]+' | sort -rn | head -1 || true)" + # 3. "n_past = N" with N>0 after a prompt-processing line (prefix kept). + n_past_n="$(printf '%s\n' "$slice" \ + | grep -aoiE 'n_past[^0-9]*([0-9]+)' \ + | grep -aoE '[0-9]+' | sort -rn | head -1 || true)" + # 4. tokens_cached / tokens from cache (some builds). + tok_cached="$(printf '%s\n' "$slice" \ + | grep -aoiE 'tokens_cached[^0-9]*([0-9]+)' \ + | grep -aoE '[0-9]+' | sort -rn | head -1 || true)" + + # ---- MISS detectors -------------------------------------------------- + # Explicit forced full re-processing (SWA / recurrent) or kv cache rm [0,. + forced_full=0 + if printf '%s\n' "$slice" | grep -aqiE 'forcing full prompt re-?processing|kv cache rm \[0,'; then + forced_full=1 + fi + + HIT=0 + why="" + if [ -n "$reused_n" ] && [ "$reused_n" -gt 0 ] 2>/dev/null; then + HIT=1; why="re-used=$reused_n" + elif [ -n "$cache_rm_start" ] && [ "$cache_rm_start" -gt 0 ] 2>/dev/null; then + HIT=1; why="kv-cache-rm-start=$cache_rm_start" + elif [ -n "$tok_cached" ] && [ "$tok_cached" -gt 0 ] 2>/dev/null; then + HIT=1; why="tokens_cached=$tok_cached" + elif [ "$forced_full" = "0" ] && [ -n "$n_past_n" ] && [ "$n_past_n" -gt 0 ] 2>/dev/null; then + # n_past>0 is the weakest signal; only trust it if nothing forced a full + # reprocess. (On a cold slot n_past tracks total processed, so it is a + # last-resort fallback per the brief.) + HIT=1; why="n_past=$n_past_n(fallback)" + fi + [ "$HIT" = "1" ] || why="${why:-no-reuse-markers (forced_full=$forced_full)}" + + OBSERVED="MISS"; [ "$HIT" = "1" ] && OBSERVED="HIT" + echo "[cache/log] observed=$OBSERVED expected=$EXPECT ($why)" + + if [ "$OBSERVED" != "$EXPECT" ]; then + echo "::error::[cache/log] KV-cache observed=$OBSERVED but expected=$EXPECT ($why). See the attribution A/B note in the workflow." + echo "---- llama-server log slice (last 60 lines) ----" + printf '%s\n' "$slice" | tail -60 + exit 1 + fi + echo "[cache/log] PASS ($OBSERVED == $EXPECT)" + exit 0 + ;; + + *) + echo "::error::unknown mode '$MODE' (want api|log|mark)" + exit 1 + ;; +esac diff --git a/.github/scripts/ci-connect-prompt.txt b/.github/scripts/ci-connect-prompt.txt new file mode 100644 index 0000000000..2d96f2b1a8 --- /dev/null +++ b/.github/scripts/ci-connect-prompt.txt @@ -0,0 +1 @@ +You are a helpful assistant in a CI connectivity check. Answer the user directly in plain text. Do not use any tools, do not take any actions, and do not explain. Just reply with the answer. diff --git a/.github/scripts/ci-min-system-prompt.txt b/.github/scripts/ci-min-system-prompt.txt new file mode 100644 index 0000000000..d55b828bbd --- /dev/null +++ b/.github/scripts/ci-min-system-prompt.txt @@ -0,0 +1 @@ +You are a coding assistant running non-interactively in a CI smoke test. Use the available file-editing and shell tools to complete the user's request directly and concisely. Do not ask questions or explain; just do the task. diff --git a/.github/scripts/serve-unsloth-run.sh b/.github/scripts/serve-unsloth-run.sh new file mode 100755 index 0000000000..34b8b962c6 --- /dev/null +++ b/.github/scripts/serve-unsloth-run.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +# +# Boot `unsloth run --disable-tools` in the background, wait for it to be +# healthy, parse the minted API key from the banner, and resolve the +# /v1/models id. Exports everything downstream steps need into $GITHUB_ENV +# (or prints it when run outside Actions). Factored out of the workflow so +# the failure-isolation logic lives in one shellcheck-clean place. +# +# Usage: +# serve-unsloth-run.sh --model REPO --gguf-variant VAR --port PORT \ +# [--gguf-file PATH] [--extra "--seed 3407 --temp 0"] \ +# [--log-dir logs] [--health-timeout 300] +# +# Why a helper and not inline YAML +# -------------------------------- +# * Every `unsloth run` invocation here is the *Unsloth server* under test. +# A failure to come up healthy is class (a) "server/API regression" and +# must be reported with a distinct `::error::` BEFORE any agent runs. +# * The banner is the documented contract a human copies from. We parse the +# exact `API Key:` line printed by unsloth_cli/commands/studio.py +# (` API Key: ` non-silent, `API Key: ` silent) so a +# silent change to that line is also caught. +# * `unsloth run` re-execs into the studio venv ($STUDIO_HOME/unsloth_studio), +# so in CI after `install.sh --local` it runs the PR's repo code. +# +# Outputs written to $GITHUB_ENV (and echoed): +# UNSLOTH_API_KEY the sk-unsloth-* key minted on the banner +# UNSLOTH_STUDIO_URL http://127.0.0.1: (so `unsloth connect` +# finds THIS server, not the hardcoded :8888) +# UNSLOTH_BASE_URL same as UNSLOTH_STUDIO_URL (alias for clarity) +# UNSLOTH_MODEL_ID the canonical id reported by /v1/models +# UNSLOTH_SERVER_PID pid of the backgrounded `unsloth run` +# UNSLOTH_LLAMA_LOG_DIR ~/.unsloth/studio/logs/llama-server + +set -uo pipefail + +# ── arg parse ──────────────────────────────────────────────────────────── +MODEL="" +GGUF_VARIANT="" +GGUF_FILE="" +PORT="" +EXTRA="" +LOG_DIR="logs" +HEALTH_TIMEOUT="300" + +while [ "$#" -gt 0 ]; do + case "$1" in + --model) MODEL="$2"; shift 2 ;; + --gguf-variant) GGUF_VARIANT="$2"; shift 2 ;; + --gguf-file) GGUF_FILE="$2"; shift 2 ;; + --port) PORT="$2"; shift 2 ;; + --extra) EXTRA="$2"; shift 2 ;; + --log-dir) LOG_DIR="$2"; shift 2 ;; + --health-timeout) HEALTH_TIMEOUT="$2"; shift 2 ;; + *) echo "serve-unsloth-run.sh: unknown arg '$1'" >&2; exit 2 ;; + esac +done + +[ -n "$PORT" ] || { echo "serve-unsloth-run.sh: --port is required" >&2; exit 2; } +if [ -z "$MODEL" ] && [ -z "$GGUF_FILE" ]; then + echo "serve-unsloth-run.sh: one of --model or --gguf-file is required" >&2 + exit 2 +fi + +mkdir -p "$LOG_DIR" +SERVER_LOG="$LOG_DIR/unsloth-run-${PORT}.log" +BASE_URL="http://127.0.0.1:${PORT}" +STUDIO_HOME_DIR="${STUDIO_HOME:-$HOME/.unsloth/studio}" +LLAMA_LOG_DIR="${STUDIO_HOME_DIR}/logs/llama-server" + +# Emit a key=value pair to $GITHUB_ENV when set, always echo for local runs. +emit() { + echo "$1=$2" + if [ -n "${GITHUB_ENV:-}" ]; then + echo "$1=$2" >> "$GITHUB_ENV" + fi +} + +server_fail() { + echo "::error::Unsloth server/API regression: $*" >&2 + echo "---- last 200 lines of $SERVER_LOG ----" >&2 + tail -200 "$SERVER_LOG" 2>/dev/null || true + exit 1 +} + +# ── port collision guard ───────────────────────────────────────────────── +# A leftover listener (or a parallel matrix cell that wandered onto our port) +# would make us attach to the wrong server and mask a real regression. Fail +# fast instead. +if command -v ss >/dev/null 2>&1; then + if ss -tln 2>/dev/null | grep -q ":${PORT}\b"; then + server_fail "port ${PORT} already has a listener before we started (collision)" + fi +fi + +# ── build the command ──────────────────────────────────────────────────── +# `unsloth run` == alias of `unsloth studio run`. --disable-tools is REQUIRED +# (passthrough mode) so the agent's own tools relay instead of the server's. +# --no-cloudflare keeps us off the network (loopback bind, no tunnel attempt). +CMD=(unsloth run -H 127.0.0.1 -p "$PORT" --disable-tools --no-cloudflare) +if [ -n "$GGUF_FILE" ]; then + CMD+=(--model "$GGUF_FILE") +else + CMD+=(--model "$MODEL") + [ -n "$GGUF_VARIANT" ] && CMD+=(--gguf-variant "$GGUF_VARIANT") +fi +# Determinism knobs + any caller passthrough (e.g. --seed 3407 --temp 0). +# shellcheck disable=SC2206 # intentional word-split of caller-controlled flags +[ -n "$EXTRA" ] && CMD+=($EXTRA) + +echo "[serve] launching: ${CMD[*]}" +echo "[serve] server log: $SERVER_LOG" + +# Run detached, no controlling TTY (setsid avoids any TTY-prompt hang and +# detaches from this step's process group so the job's teardown is clean). +setsid "${CMD[@]}" > "$SERVER_LOG" 2>&1 < /dev/null & +SERVER_PID=$! +emit UNSLOTH_SERVER_PID "$SERVER_PID" + +# ── wait for /api/health == healthy ────────────────────────────────────── +HEALTHY=0 +for _ in $(seq 1 "$HEALTH_TIMEOUT"); do + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + server_fail "process exited before becoming healthy (pid $SERVER_PID)" + fi + if curl -fs "${BASE_URL}/api/health" -o "$LOG_DIR/health-${PORT}.json" 2>/dev/null; then + if jq -e '.status == "healthy"' "$LOG_DIR/health-${PORT}.json" >/dev/null 2>&1; then + HEALTHY=1 + break + fi + fi + sleep 1 +done +[ "$HEALTHY" = "1" ] || server_fail "did not report /api/health healthy within ${HEALTH_TIMEOUT}s" +echo "[serve] /api/health healthy" + +# ── parse the API key from the banner ──────────────────────────────────── +# Match both the non-silent " API Key: " and silent "API Key: " +# forms. We do NOT trust a fixed column count; we take the sk-unsloth-* token. +API_KEY="" +for _ in $(seq 1 30); do + API_KEY="$(grep -aoE 'sk-unsloth-[A-Za-z0-9_-]+' "$SERVER_LOG" 2>/dev/null | head -1 || true)" + [ -n "$API_KEY" ] && break + sleep 1 +done +if [ -z "$API_KEY" ]; then + # Fallback: take whatever follows an "API Key:" label, in case the key + # prefix scheme changes. Still a parse-fragility guard, not silent. + API_KEY="$(grep -aE 'API Key:' "$SERVER_LOG" 2>/dev/null \ + | sed -E 's/.*API Key:[[:space:]]*//' | head -1 || true)" +fi +[ -n "$API_KEY" ] || server_fail "could not parse an API key from the banner (banner-parse fragility -- check the 'API Key:' line in unsloth_cli/commands/studio.py)" +echo "::add-mask::${API_KEY}" +emit UNSLOTH_API_KEY "$API_KEY" + +# ── resolve /v1/models id ──────────────────────────────────────────────── +if ! curl -fs "${BASE_URL}/v1/models" \ + -H "Authorization: Bearer ${API_KEY}" -o "$LOG_DIR/models-${PORT}.json" 2>/dev/null; then + server_fail "/v1/models did not respond (or rejected the banner key)" +fi +MODEL_ID="$(jq -r '.data[0].id // empty' "$LOG_DIR/models-${PORT}.json" 2>/dev/null || true)" +[ -n "$MODEL_ID" ] || server_fail "/v1/models returned no model id (model failed to load)" +echo "[serve] resolved model id: $MODEL_ID" + +emit UNSLOTH_MODEL_ID "$MODEL_ID" +emit UNSLOTH_STUDIO_URL "$BASE_URL" +emit UNSLOTH_BASE_URL "$BASE_URL" +emit UNSLOTH_LLAMA_LOG_DIR "$LLAMA_LOG_DIR" + +echo "[serve] server is up: ${BASE_URL} (model ${MODEL_ID})" diff --git a/.github/workflows/local-agent-guides-ci.yml b/.github/workflows/local-agent-guides-ci.yml new file mode 100644 index 0000000000..150dbc3fde --- /dev/null +++ b/.github/workflows/local-agent-guides-ci.yml @@ -0,0 +1,599 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +# Local Agent Guides CI +# ===================== +# Detects when our local-agent setup recipes drift out of sync with +# `unsloth run`. Boots a real `unsloth run --disable-tools` server and +# drives the coding agents end to end through the *exact* recipes defined +# in unsloth_cli/commands/connect.py (the in-repo source of truth -- there +# is no docs/ tree). Wherever connect.py has a recipe we drive the agent +# via `unsloth connect --no-launch` and execute what it prints, so +# the test self-updates against connect.py and catches silent recipe drift. +# +# Source-of-truth files this workflow guards: +# unsloth_cli/commands/connect.py the `unsloth connect ` recipes +# unsloth_cli/commands/studio.py the `unsloth run` banner (API Key line) +# +# Failure taxonomy (each surfaced with a distinct ::error:: + the agent name +# + the connect.py location, so a red X is immediately triageable): +# (a) Unsloth server/API regression -- the dialect HTTP preflight fails +# BEFORE the agent runs (or the server never becomes healthy). +# (b) Agent package install failed -- npm/curl install of the CLI failed. +# (c) Guide drift -- preflight passed + install ok, but +# the documented `unsloth connect` flow produced no/garbled output. +# +# Agents covered (6): claude, codex, hermes, openclaw, opencode, pi. +# - claude/codex/hermes/openclaw/opencode have a connect.py recipe. +# - pi has NO `unsloth connect pi` command in connect.py at HEAD; it is +# driven by a hand-written recipe and the matrix cell asserts that the +# missing connect recipe is the (known) reason, so the day connect.py +# grows a `pi` command this cell flips to the self-updating path. + +name: Local Agent Guides CI + +on: + # Off-peak weekly, deliberately a NON-:00 minute to dodge the top-of-hour + # GitHub-hosted-runner stampede. + schedule: + - cron: '37 7 * * 1' + workflow_dispatch: + pull_request: + paths: + - 'unsloth_cli/**' + - 'studio/backend/routes/**' + # Contracts this workflow asserts that live outside routes/**: the + # /api/health endpoint, the llama-server KV-cache log behavior, and the + # request/response schemas the agent dialects depend on. + - 'studio/backend/main.py' + - 'studio/backend/core/inference/llama_cpp.py' + - 'studio/backend/models/**' + - 'install.sh' + - '.github/workflows/local-agent-guides-ci.yml' + - '.github/scripts/serve-unsloth-run.sh' + - '.github/scripts/assert-prompt-cache.sh' + - '.github/scripts/agent-guides-install.sh' + - '.github/scripts/agent-guides-drive.sh' + - '.github/scripts/ci-connect-prompt.txt' + - '.github/scripts/ci-min-system-prompt.txt' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # Determinism precedent (studio-inference-smoke.yml): temp 0 + fixed seed. + UNSLOTH_SEED: '3407' + # A single invoke must never hang the runner on a headless TTY prompt. With + # prefill-shrinking flags (minimal system prompt + restricted tools) a turn on + # a 4B model finishes in a couple of minutes on CPU; this also caps how long a + # still-large-prompt agent burns before failing. Well under the 6h job cap. + AGENT_INVOKE_TIMEOUT: '600' + +jobs: + # ═════════════════════════════════════════════════════════════════════ + # Job 1: connection + # Per-agent: serve gemma-3-270m, HTTP-preflight the agent's dialect, + # install the agent, run `unsloth connect --no-launch`, execute + # the emitted recipe with a trivial prompt, assert a non-empty reply. + # Runs on PR + weekly + dispatch. Each matrix cell is its own runner so + # it serves exactly one model on its own port. + # ═════════════════════════════════════════════════════════════════════ + connection: + name: connection (${{ matrix.agent }}) + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + agent: [claude, codex, hermes, openclaw, opencode, pi] + include: + # OpenClaw needs Node 24; everything else is happy on 22. + - agent: openclaw + node: '24' + env: + # gemma-4-E4B (128K context, capable enough to drive every agent for a + # trivial reply; the 270m model produced empty/failed responses for + # codex/openclaw and is below hermes' 64K context floor). Served as a flat + # GGUF file (the -MTP- repo ships no separate draft, so this is plain 4B). + GGUF_REPO: unsloth/gemma-4-E4B-it-GGUF + GGUF_FILE: gemma-4-E4B-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18901' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Linux deps for llama.cpp prebuilt + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev libssl-dev jq + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ matrix.node || '22' }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + cache: 'pip' + + - name: Restore GGUF model file + id: cache-gguf + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Download GGUF if cache miss + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + python -m pip install --upgrade huggingface_hub + mkdir -p gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache + + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Install Studio (--local, --no-torch) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + mkdir -p logs + set -o pipefail + bash install.sh --local --no-torch 2>&1 | tee logs/install.log + + # ── boot the server under test (factored helper) ────────────────── + - name: Serve unsloth run --disable-tools (gemma-4-E4B) + run: | + unsloth studio reset-password + bash .github/scripts/serve-unsloth-run.sh \ + --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ + --port "$STUDIO_PORT" --log-dir logs \ + --extra "--seed $UNSLOTH_SEED --temp 0" \ + --health-timeout 900 + + # ── (a) server/API preflight: prove the dialect works BEFORE the agent ─ + # Distinct error class. If this step fails it is a SERVER regression, + # not the agent's or the guide's fault, and the agent steps never run. + - name: Preflight the agent's API dialect (class-a isolation) + env: + AGENT: ${{ matrix.agent }} + run: | + set -uo pipefail + B="$UNSLOTH_BASE_URL"; K="$UNSLOTH_API_KEY" + preflight_fail() { + echo "::error::[server/API regression] agent=$AGENT: $* (preflight failed BEFORE install/connect; this is class (a), not guide drift). Endpoint contract lives in studio/backend/routes/**."; + exit 1 + } + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/models" \ + -H "Authorization: Bearer $K") || true + [ "$code" = "200" ] || preflight_fail "/v1/models returned HTTP $code" + case "$AGENT" in + claude) + # Anthropic Messages dialect. + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/messages" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/messages returned HTTP $code" + ;; + codex) + # Codex always streams /v1/responses. + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/responses" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"input\":\"Hi\",\"max_output_tokens\":16,\"stream\":true}") || true + [ "$code" = "200" ] || preflight_fail "/v1/responses returned HTTP $code" + ;; + *) + # OpenAI Chat Completions dialect (hermes/opencode/pi/openclaw). + # OpenClaw's connect.py recipe writes an "openai-completions" + # provider (write_openclaw_config), so it uses this path, not + # /v1/messages. + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/chat/completions" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/chat/completions returned HTTP $code" + ;; + esac + echo "preflight OK for $AGENT" + + # ── (b) install the agent CLI (hardened npm/curl, retried) ───────── + - name: Install agent CLI (class-b isolation) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-install.sh "$AGENT" + + # ── (c) drive the agent via connect.py and assert a reply ────────── + # For the 5 agents with a connect.py recipe we run + # `unsloth connect --no-launch`, eval its env/unset exports, + # then run the printed command with a hard timeout (no headless-TTY + # hang). Pi has no connect recipe, so it is driven by hand and the + # cell asserts that absence is the (known) reason. + - name: Drive ${{ matrix.agent }} via unsloth connect (class-c isolation) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-drive.sh connection "$AGENT" + + - name: Collect server logs (debug) + if: always() + run: | + mkdir -p logs/studio-logs + cp -r "$HOME/.unsloth/studio/logs/." logs/studio-logs/ 2>/dev/null || true + # Redact the key across the WHOLE logs/ tree, not just studio-logs: + # serve-unsloth-run.sh records the `unsloth run` banner (which prints + # `API Key: `) into logs/unsloth-run-.log, and the upload + # step publishes all of logs/, so scrubbing only studio-logs would leak + # the bearer token in the retained artifact. + if [ -n "${UNSLOTH_API_KEY:-}" ]; then + grep -rlF "$UNSLOTH_API_KEY" logs 2>/dev/null | while IFS= read -r f; do + sed -i "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + done + fi + + - name: Stop Studio + if: always() + run: | + # Guard the PID: an unset/zero UNSLOTH_SERVER_PID would make + # `kill 0` signal this step's whole process group and abort cleanup. + if [ -n "${UNSLOTH_SERVER_PID:-}" ] && [ "${UNSLOTH_SERVER_PID}" != "0" ]; then + kill "${UNSLOTH_SERVER_PID}" 2>/dev/null || true + fi + sleep 2 + ss -tln 2>/dev/null | grep ":${STUDIO_PORT}" || true + + - name: Upload logs + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: connection-${{ matrix.agent }}-log + path: | + logs/ + redacted-configs/ + retention-days: 7 + + # ═════════════════════════════════════════════════════════════════════ + # Job 2: file-edit + # The deterministic 2-turn hello.py test on Qwen3.5-4B (smaller models + # can't reliably drive the heavyweight agents' edit flows). Weekly + + # dispatch only -- it is the slow, model-heavy job and must not gate PRs. + # ═════════════════════════════════════════════════════════════════════ + file-edit: + name: file-edit (${{ matrix.agent }}) + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 60 + # hermes and openclaw drive a multi-turn tool loop that a CPU-only runner + # cannot finish in time (e.g. openclaw holds its 300s session-write-lock past + # expiry; each turn re-prefills the tool prompt at ~16 tok/s). Their endpoint + # wiring + generation are already hard-gated by the connection job, so the + # file-edit cell is best-effort here -- it still runs and uploads logs, but a + # timeout does not fail the workflow. Drop best_effort (or move e2e to a GPU + # runner) to make it blocking again. + continue-on-error: ${{ matrix.best_effort || false }} + strategy: + fail-fast: false + matrix: + agent: [claude, codex, hermes, openclaw, opencode, pi] + include: + - agent: openclaw + node: '24' + best_effort: true + - agent: hermes + best_effort: true + env: + # gemma-4-E4B served as a flat GGUF file (cache size tracks the .gguf 1:1, + # no xet-chunk inflation; the -MTP- repo ships no separate draft file). + GGUF_REPO: unsloth/gemma-4-E4B-it-GGUF + GGUF_FILE: gemma-4-E4B-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18902' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Linux deps for llama.cpp prebuilt + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev libssl-dev jq + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: ${{ matrix.node || '22' }} + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + cache: 'pip' + + - name: Restore GGUF model file + id: cache-gguf + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Download GGUF if cache miss + id: download-gguf + if: steps.cache-gguf.outputs.cache-hit != 'true' || steps.cache-gguf.outcome != 'success' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + python -m pip install --upgrade huggingface_hub + mkdir -p gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache + + - name: Save GGUF model file + if: always() && steps.download-gguf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: gguf-cache + key: ${{ runner.os }}-gguf-${{ env.GGUF_REPO }}-${{ env.GGUF_FILE }}-v1 + + - name: Install Studio (--local, --no-torch) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + mkdir -p logs + set -o pipefail + bash install.sh --local --no-torch 2>&1 | tee logs/install.log + + - name: Serve unsloth run --disable-tools (gemma-4-E4B) + run: | + unsloth studio reset-password + bash .github/scripts/serve-unsloth-run.sh \ + --gguf-file "$GITHUB_WORKSPACE/gguf-cache/${GGUF_FILE}" \ + --port "$STUDIO_PORT" --log-dir logs \ + --extra "--seed $UNSLOTH_SEED --temp 0" \ + --health-timeout 900 + + - name: Preflight the agent's API dialect (class-a isolation) + env: + AGENT: ${{ matrix.agent }} + run: | + set -uo pipefail + B="$UNSLOTH_BASE_URL"; K="$UNSLOTH_API_KEY" + preflight_fail() { + echo "::error::[server/API regression] agent=$AGENT: $* (preflight failed BEFORE install/connect; this is class (a), not guide drift). Endpoint contract lives in studio/backend/routes/**."; + exit 1 + } + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/models" \ + -H "Authorization: Bearer $K") || true + [ "$code" = "200" ] || preflight_fail "/v1/models returned HTTP $code" + # Probe the same dialect the agent will use, so a streaming/messages + # regression in the weekly run is reported as class (a) here instead of + # surfacing later as guide drift (mirrors the connection job). + case "$AGENT" in + claude) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/messages" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/messages returned HTTP $code" + ;; + codex) + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/responses" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"input\":\"Hi\",\"max_output_tokens\":16,\"stream\":true}") || true + [ "$code" = "200" ] || preflight_fail "/v1/responses returned HTTP $code" + ;; + *) + # OpenAI Chat Completions dialect (hermes/opencode/pi/openclaw). + code=$(curl -s -o /tmp/pf.json -w '%{http_code}' "$B/v1/chat/completions" \ + -H "Authorization: Bearer $K" -H 'content-type: application/json' \ + --max-time 120 \ + -d "{\"model\":\"$UNSLOTH_MODEL_ID\",\"max_tokens\":16,\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"}]}") || true + [ "$code" = "200" ] || preflight_fail "/v1/chat/completions returned HTTP $code" + ;; + esac + echo "preflight OK for $AGENT" + + - name: Install agent CLI (class-b isolation) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-install.sh "$AGENT" + + - name: 2-turn hello.py test (class-c isolation) + env: + AGENT: ${{ matrix.agent }} + run: bash .github/scripts/agent-guides-drive.sh file-edit "$AGENT" + + - name: Collect server logs (debug) + if: always() + run: | + mkdir -p logs/studio-logs + cp -r "$HOME/.unsloth/studio/logs/." logs/studio-logs/ 2>/dev/null || true + # Redact the key across the WHOLE logs/ tree, not just studio-logs: + # serve-unsloth-run.sh records the `unsloth run` banner (which prints + # `API Key: `) into logs/unsloth-run-.log, and the upload + # step publishes all of logs/, so scrubbing only studio-logs would leak + # the bearer token in the retained artifact. + if [ -n "${UNSLOTH_API_KEY:-}" ]; then + grep -rlF "$UNSLOTH_API_KEY" logs 2>/dev/null | while IFS= read -r f; do + sed -i "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + done + fi + + - name: Stop Studio + if: always() + run: | + # Guard the PID: an unset/zero UNSLOTH_SERVER_PID would make + # `kill 0` signal this step's whole process group and abort cleanup. + if [ -n "${UNSLOTH_SERVER_PID:-}" ] && [ "${UNSLOTH_SERVER_PID}" != "0" ]; then + kill "${UNSLOTH_SERVER_PID}" 2>/dev/null || true + fi + sleep 2 + ss -tln 2>/dev/null | grep ":${STUDIO_PORT}" || true + + - name: Upload logs + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: file-edit-${{ matrix.agent }}-log + path: | + logs/ + agent-workdir/ + redacted-configs/ + retention-days: 7 + + # ═════════════════════════════════════════════════════════════════════ + # Job 3: prompt-cache + # (a) curl 2-turn /v1/chat/completions: assert turn-2 cached_tokens > 0 + # (server prompt-cache sanity). + # (b) Claude Code attribution A/B: with CLAUDE_CODE_ATTRIBUTION_HEADER=0 + # expect a llama-server KV-cache HIT on turn 2; without it expect a + # MISS. If it inverts, the guide flag is stale. + # PR + weekly + dispatch (cheap, gemma-3-270m). + # ═════════════════════════════════════════════════════════════════════ + prompt-cache: + name: prompt-cache (gemma-3-270m) + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + GGUF_REPO: unsloth/gemma-3-270m-it-GGUF + GGUF_VARIANT: UD-Q4_K_XL + GGUF_FILE: gemma-3-270m-it-UD-Q4_K_XL.gguf + STUDIO_PORT: '18903' + HF_HOME: ${{ github.workspace }}/hf-cache + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Linux deps for llama.cpp prebuilt + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + libcurl4-openssl-dev libssl-dev jq + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22' + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + cache: 'pip' + + - name: Restore HF_HOME for ${{ env.GGUF_REPO }} + id: cache-hf + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + continue-on-error: true + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 + + - name: Prime HF_HOME with the GGUF + id: prime-hf + if: steps.cache-hf.outputs.cache-hit != 'true' || steps.cache-hf.outcome != 'success' + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + python -m pip install --upgrade huggingface_hub + mkdir -p hf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" + + - name: Save HF_HOME for ${{ env.GGUF_REPO }} + if: always() && steps.prime-hf.outcome == 'success' + uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: hf-cache + key: ${{ runner.os }}-hf-${{ env.GGUF_REPO }}-${{ env.GGUF_VARIANT }}-v2 + + - name: Install Studio (--local, --no-torch) + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: | + mkdir -p logs + set -o pipefail + bash install.sh --local --no-torch 2>&1 | tee logs/install.log + + - name: Serve unsloth run --disable-tools (gemma-3-270m) + run: | + unsloth studio reset-password + bash .github/scripts/serve-unsloth-run.sh \ + --model "$GGUF_REPO" --gguf-variant "$GGUF_VARIANT" \ + --port "$STUDIO_PORT" --log-dir logs \ + --extra "--seed $UNSLOTH_SEED --temp 0" + + # (a) server prompt-cache sanity on the OpenAI chat path. The helper runs + # the 2-turn probe internally (turn 2 reuses turn 1's prefix) and asserts + # turn-2 usage.prompt_tokens_details.cached_tokens > 0. This is the hard + # gate -- it proves llama.cpp KV reuse is surfaced on /v1/chat/completions. + - name: Server prompt-cache sanity (cached_tokens > 0) + run: bash .github/scripts/assert-prompt-cache.sh api "$UNSLOTH_BASE_URL" "$UNSLOTH_API_KEY" + + - name: Install Claude Code (class-b isolation) + env: + AGENT: claude + run: bash .github/scripts/agent-guides-install.sh claude + + # (b) Claude attribution A/B against the llama-server log. This is the most + # environment-sensitive check (it depends on the bundled llama.cpp's + # slot-reuse log wording and on claude --continue reusing the prefix), so + # it is non-blocking until calibrated on the first scheduled run; the + # server cache sanity above is the hard gate. The step still prints the + # observed HIT/MISS so drift is visible in the log + artifacts. + - name: Claude attribution A/B (HIT with header=0, MISS without) + continue-on-error: true + run: bash .github/scripts/agent-guides-drive.sh attribution-ab claude + + - name: Collect server logs (debug) + if: always() + run: | + mkdir -p logs/studio-logs + cp -r "$HOME/.unsloth/studio/logs/." logs/studio-logs/ 2>/dev/null || true + # Redact the key across the WHOLE logs/ tree, not just studio-logs: + # serve-unsloth-run.sh records the `unsloth run` banner (which prints + # `API Key: `) into logs/unsloth-run-.log, and the upload + # step publishes all of logs/, so scrubbing only studio-logs would leak + # the bearer token in the retained artifact. + if [ -n "${UNSLOTH_API_KEY:-}" ]; then + grep -rlF "$UNSLOTH_API_KEY" logs 2>/dev/null | while IFS= read -r f; do + sed -i "s#${UNSLOTH_API_KEY}##g" "$f" 2>/dev/null || true + done + fi + + - name: Stop Studio + if: always() + run: | + # Guard the PID: an unset/zero UNSLOTH_SERVER_PID would make + # `kill 0` signal this step's whole process group and abort cleanup. + if [ -n "${UNSLOTH_SERVER_PID:-}" ] && [ "${UNSLOTH_SERVER_PID}" != "0" ]; then + kill "${UNSLOTH_SERVER_PID}" 2>/dev/null || true + fi + sleep 2 + ss -tln 2>/dev/null | grep ":${STUDIO_PORT}" || true + + - name: Upload logs + if: always() + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: prompt-cache-log + path: | + logs/ + redacted-configs/ + retention-days: 7