148 lines
No EOL
6.3 KiB
Bash
148 lines
No EOL
6.3 KiB
Bash
#!/usr/bin/env bash
|
|
# Built standing on the shoulders of billions of dwarves
|
|
# Created by John A. Hoeven with the ethical assistance of Claude AI
|
|
# Licence: The Unlicense — https://unlicense.org
|
|
# ---------------------------------------------------------------------------
|
|
# llama-cpp-promote.sh
|
|
# /opt/llama-cpp/scripts/llama-cpp-promote.sh
|
|
# Version: v0.0.1 | Status: DEVELOPMENT
|
|
# Role: Orchestrator
|
|
# ---------------------------------------------------------------------------
|
|
# Purpose: Point the 'current' symlink at a given staged release and
|
|
# restart llama-server. Used for BOTH forward promotion (new
|
|
# monthly build) and rollback (point back at the previous tag) —
|
|
# the operation is identical either way, only the target tag
|
|
# differs. Verifies the API responds before declaring success;
|
|
# auto-reverts the symlink if the new build fails to come up.
|
|
# Target: BigBoy — AlmaLinux 10.2, systemd-managed llama-server.service
|
|
# Entry: sudo ./llama-cpp-promote.sh <build-tag>
|
|
# e.g. sudo ./llama-cpp-promote.sh b9985 (promote)
|
|
# sudo ./llama-cpp-promote.sh b9840 (roll back)
|
|
# Depends: curl, systemctl
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# No 'set -e' — every operation is checked explicitly (CE OS standard §1.8).
|
|
|
|
# ── Configuration ────────────────────────────────────────────────────────
|
|
|
|
LLAMA_ROOT="/opt/llama-cpp"
|
|
RELEASES_DIR="${LLAMA_ROOT}/releases"
|
|
CURRENT_LINK="${LLAMA_ROOT}/current"
|
|
SERVICE_NAME="llama-server.service"
|
|
API_PORT="8080"
|
|
VERIFY_TIMEOUT_SECS=30
|
|
LOG_DIR="${HOME}/.local/logs/bigboy-llama-update"
|
|
LOG_FILE="${LOG_DIR}/promote-$(date '+%Y-%m-%d_%H%M%S').log"
|
|
|
|
TARGET_TAG="${1:-}"
|
|
|
|
# ── Logging ──────────────────────────────────────────────────────────────
|
|
|
|
mkdir -p "${LOG_DIR}"
|
|
|
|
_log() {
|
|
local level="${1}"
|
|
local msg="${2}"
|
|
local ts
|
|
ts="$(date '+%Y-%m-%d %H:%M:%S')"
|
|
printf '[%s] [%s] %s\n' "${ts}" "${level}" "${msg}" | tee -a "${LOG_FILE}"
|
|
}
|
|
log_info() { _log "INFO " "${1}"; }
|
|
log_warn() { _log "WARN " "${1}" >&2; }
|
|
log_error() { _log "ERROR" "${1}" >&2; }
|
|
|
|
# ── Pre-flight checks ────────────────────────────────────────────────────
|
|
|
|
if [[ "${EUID}" -eq 0 ]]; then
|
|
log_error "Do not run this script directly as root — it invokes sudo"
|
|
log_error "only for the specific steps that need it."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${TARGET_TAG}" ]]; then
|
|
log_error "Usage: ${0} <build-tag>"
|
|
log_error "Available staged releases:"
|
|
ls -1 "${RELEASES_DIR}" 2>/dev/null | sed 's/^/ /' | tee -a "${LOG_FILE}"
|
|
exit 1
|
|
fi
|
|
|
|
TARGET_PATH="${RELEASES_DIR}/${TARGET_TAG}"
|
|
|
|
if [[ ! -x "${TARGET_PATH}/build/bin/llama-server" ]]; then
|
|
log_error "No valid build found at ${TARGET_PATH}/build/bin/llama-server"
|
|
log_error "Run llama-cpp-update.sh ${TARGET_TAG} first if this is a new tag."
|
|
exit 1
|
|
fi
|
|
|
|
PREVIOUS_TAG=""
|
|
if [[ -L "${CURRENT_LINK}" ]]; then
|
|
PREVIOUS_TAG="$(basename "$(readlink -f "${CURRENT_LINK}")")"
|
|
fi
|
|
|
|
if [[ "${PREVIOUS_TAG}" == "${TARGET_TAG}" ]]; then
|
|
log_warn "current already points at ${TARGET_TAG} — nothing to do."
|
|
exit 0
|
|
fi
|
|
|
|
# ── Confirmation ─────────────────────────────────────────────────────────
|
|
|
|
printf '\nCurrent live release: %s\n' "${PREVIOUS_TAG:-none}"
|
|
printf 'Switching to: %s\n' "${TARGET_TAG}"
|
|
printf '%s will be restarted.\n' "${SERVICE_NAME}"
|
|
printf '\nProceed? [y/N] '
|
|
read -r response
|
|
case "${response}" in
|
|
[yY]|[yY][eE][sS]) ;;
|
|
*) log_info "Aborted by user."; exit 0 ;;
|
|
esac
|
|
|
|
# ── Step 1: swap symlink ─────────────────────────────────────────────────
|
|
|
|
log_info "Repointing ${CURRENT_LINK} -> ${TARGET_PATH}"
|
|
if ! sudo ln -sfn "${TARGET_PATH}" "${CURRENT_LINK}"; then
|
|
log_error "Failed to update symlink. Service left untouched — no restart attempted."
|
|
exit 1
|
|
fi
|
|
|
|
# ── Step 2: restart service ───────────────────────────────────────────────
|
|
|
|
log_info "Restarting ${SERVICE_NAME}"
|
|
if ! sudo systemctl restart "${SERVICE_NAME}" >>"${LOG_FILE}" 2>&1; then
|
|
log_error "Service failed to restart. Rolling symlink back to ${PREVIOUS_TAG}."
|
|
if [[ -n "${PREVIOUS_TAG}" ]]; then
|
|
sudo ln -sfn "${RELEASES_DIR}/${PREVIOUS_TAG}" "${CURRENT_LINK}"
|
|
sudo systemctl restart "${SERVICE_NAME}" >>"${LOG_FILE}" 2>&1
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
# ── Step 3: verify the API actually responds ─────────────────────────────
|
|
|
|
log_info "Waiting for API on port ${API_PORT} (up to ${VERIFY_TIMEOUT_SECS}s)"
|
|
elapsed=0
|
|
until curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:${API_PORT}/health" \
|
|
2>/dev/null | grep -q '200'; do
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
if [[ "${elapsed}" -ge "${VERIFY_TIMEOUT_SECS}" ]]; then
|
|
log_error "API did not respond within ${VERIFY_TIMEOUT_SECS}s."
|
|
log_error "Auto-reverting to previous release: ${PREVIOUS_TAG:-none}"
|
|
if [[ -n "${PREVIOUS_TAG}" ]]; then
|
|
sudo ln -sfn "${RELEASES_DIR}/${PREVIOUS_TAG}" "${CURRENT_LINK}"
|
|
sudo systemctl restart "${SERVICE_NAME}" >>"${LOG_FILE}" 2>&1
|
|
log_warn "Reverted. Investigate ${TARGET_TAG} before retrying."
|
|
fi
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
log_info "API confirmed healthy on ${TARGET_TAG}."
|
|
log_info "Promotion complete: ${PREVIOUS_TAG:-none} -> ${TARGET_TAG}"
|
|
log_info ""
|
|
log_info "Previous release (${PREVIOUS_TAG:-none}) left in place at:"
|
|
log_info " ${RELEASES_DIR}/${PREVIOUS_TAG}"
|
|
log_info "Recommended: keep it for at least a week before removing, in case"
|
|
log_info "an issue surfaces after a few days of real use rather than immediately."
|
|
log_info ""
|
|
log_info "To roll back at any point:"
|
|
log_info " sudo ./llama-cpp-promote.sh ${PREVIOUS_TAG}" |