#!/usr/bin/env bash
# Built standing on the shoulders of billions of dwarves
# Created by John A. Hoeven with the ethical assistance of Claude AI
# Licence: The Unlicense — https://unlicense.org
# ---------------------------------------------------------------------------
# llama-cpp-update.sh
# /opt/llama-cpp/scripts/llama-cpp-update.sh
# Version: v0.0.1 | Status: DEVELOPMENT
# Role:     Orchestrator
# ---------------------------------------------------------------------------
# Purpose:  Build and stage a new pinned llama.cpp release on BigBoy, behind
#           a btrfs snapshot safety net and a versioned-symlink release
#           layout. Does NOT promote the new build automatically — that is
#           a separate, deliberate step (see llama-cpp-promote.sh) taken
#           only after Phase 12-style validation passes.
# Target:   BigBoy — AlmaLinux 10.2, @llama-cpp btrfs subvolume at
#           /opt/llama-cpp, NVIDIA RTX 5060 Ti (sm_120)
# Entry:    sudo ./llama-cpp-update.sh <build-tag>
#           e.g. sudo ./llama-cpp-update.sh b9985
# Depends:  git, cmake, gcc, nvcc (CUDA toolkit), btrfs-progs
# Note:     This is a standalone ops script for one Alma server, not part
#           of the CEOS multi-distro installer framework — so it does not
#           use ce_env.conf / pkg_* / CE_PRIV. Privilege is via sudo
#           directly, detected explicitly below rather than assumed.
# ---------------------------------------------------------------------------

# No 'set -e' — every operation is checked explicitly (CE OS standard §1.8).

# ── Configuration ────────────────────────────────────────────────────────

LLAMA_ROOT="/opt/llama-cpp"
RELEASES_DIR="${LLAMA_ROOT}/releases"
SNAPSHOT_DIR="/opt/.snapshots/llama-cpp"
REPO_URL="https://github.com/ggml-org/llama.cpp"
CUDA_ARCH="120"          # RTX 5060 Ti — Blackwell sm_120
LOG_DIR="${HOME}/.local/logs/bigboy-llama-update"
LOG_FILE="${LOG_DIR}/update-$(date '+%Y-%m-%d_%H%M%S').log"

BUILD_TAG="${1:-}"

# ── Logging ──────────────────────────────────────────────────────────────

mkdir -p "${LOG_DIR}"

_log() {
    local level="${1}"
    local msg="${2}"
    local ts
    ts="$(date '+%Y-%m-%d %H:%M:%S')"
    printf '[%s] [%s] %s\n' "${ts}" "${level}" "${msg}" | tee -a "${LOG_FILE}"
}
log_info()  { _log "INFO " "${1}"; }
log_warn()  { _log "WARN " "${1}" >&2; }
log_error() { _log "ERROR" "${1}" >&2; }

# ── Cleanup ──────────────────────────────────────────────────────────────

BUILD_TMP=""

cleanup() {
    local exit_code="${?}"
    if [[ -n "${BUILD_TMP}" && -d "${BUILD_TMP}" && "${exit_code}" -ne 0 ]]; then
        log_warn "Non-zero exit — leaving ${BUILD_TMP} in place for inspection."
        log_warn "Remove it manually once reviewed: rm -rf ${BUILD_TMP}"
    fi
    log_info "Run finished. Log: ${LOG_FILE}"
}
trap cleanup EXIT INT TERM HUP

# ── Pre-flight checks ────────────────────────────────────────────────────

if [[ "${EUID}" -eq 0 ]]; then
    log_error "Do not run this script directly as root. Run as your normal"
    log_error "user; it will invoke sudo only for the specific steps that need it."
    exit 1
fi

if [[ -z "${BUILD_TAG}" ]]; then
    log_error "Usage: ${0} <build-tag>   e.g. ${0} b9985"
    log_error "Check https://github.com/ggml-org/llama.cpp/releases for current tags."
    exit 1
fi

if [[ -d "${RELEASES_DIR}/${BUILD_TAG}" ]]; then
    log_error "Release ${BUILD_TAG} already exists at ${RELEASES_DIR}/${BUILD_TAG}"
    log_error "Remove it first if you intend to rebuild this tag."
    exit 1
fi

for tool in git cmake nvcc; do
    if ! command -v "${tool}" >/dev/null 2>&1; then
        log_error "Required tool not found: ${tool}"
        exit 1
    fi
done

# ── Confirmation ─────────────────────────────────────────────────────────

printf '\nAbout to build llama.cpp %s into %s\n' "${BUILD_TAG}" "${RELEASES_DIR}/${BUILD_TAG}"
printf 'A btrfs snapshot of %s will be taken first.\n' "${LLAMA_ROOT}"
printf 'This will NOT restart the live service — that is a separate promote step.\n'
printf '\nProceed? [y/N] '
read -r response
case "${response}" in
    [yY]|[yY][eE][sS]) ;;
    *) log_info "Aborted by user."; exit 0 ;;
esac

# ── Step 1: btrfs snapshot (safety net) ──────────────────────────────────

mkdir -p "${SNAPSHOT_DIR}"
SNAPSHOT_PATH="${SNAPSHOT_DIR}/pre-${BUILD_TAG}-$(date '+%Y%m%d')"

log_info "Taking btrfs snapshot: ${SNAPSHOT_PATH}"
if ! sudo btrfs subvolume snapshot -r "${LLAMA_ROOT}" "${SNAPSHOT_PATH}" \
        >>"${LOG_FILE}" 2>&1; then
    log_error "Snapshot failed. Aborting before touching anything else."
    exit 1
fi
log_info "Snapshot confirmed at ${SNAPSHOT_PATH}"

# ── Step 2: clone and checkout pinned tag ────────────────────────────────

BUILD_TMP="$(mktemp -d)"
log_info "Cloning llama.cpp (shallow, tag-only) into ${BUILD_TMP}"

# Shallow clone directly at the target tag — we only ever need this one
# pinned commit, not full history. Cuts clone size roughly in half versus
# a full clone + checkout, which matters once you're keeping many releases.
if ! git clone --quiet --depth 1 --branch "${BUILD_TAG}" \
        "${REPO_URL}" "${BUILD_TMP}/llama.cpp" >>"${LOG_FILE}" 2>&1; then
    log_error "Shallow clone of tag ${BUILD_TAG} failed."
    log_error "Confirm the tag exists upstream: ${REPO_URL}/releases"
    exit 1
fi

# ── Step 3: build (standard flags only — no exotic tuning, see notes) ────

log_info "Configuring CMake build (CUDA, sm_${CUDA_ARCH}, Release)"
if ! (cd "${BUILD_TMP}/llama.cpp" && cmake -B build \
        -DGGML_CUDA=ON \
        -DCMAKE_BUILD_TYPE=Release \
        -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCH}") >>"${LOG_FILE}" 2>&1; then
    log_error "CMake configure failed. See ${LOG_FILE} for details."
    exit 1
fi

log_info "Building — this takes a while, watch ${LOG_FILE} for progress"
if ! (cd "${BUILD_TMP}/llama.cpp" && cmake --build build --config Release -j) \
        >>"${LOG_FILE}" 2>&1; then
    log_error "Build failed. See ${LOG_FILE} for details."
    log_error "Known issue: MXFP4 quant kernels can fail to compile on sm_120."
    log_error "This build does not require MXFP4 — check for unrelated errors first."
    exit 1
fi

if [[ ! -x "${BUILD_TMP}/llama.cpp/build/bin/llama-server" ]]; then
    log_error "Build reported success but llama-server binary is missing. Aborting."
    exit 1
fi

# ── Step 4: stage into versioned release directory ──────────────────────

log_info "Staging build into ${RELEASES_DIR}/${BUILD_TAG}"
sudo mkdir -p "${RELEASES_DIR}"
if ! sudo cp -a "${BUILD_TMP}/llama.cpp" "${RELEASES_DIR}/${BUILD_TAG}" \
        >>"${LOG_FILE}" 2>&1; then
    log_error "Failed to stage build into ${RELEASES_DIR}/${BUILD_TAG}"
    exit 1
fi

log_info "Build ${BUILD_TAG} staged successfully."
log_info "Current live release is untouched — 'current' symlink not modified."
log_info "Next step: run Phase 12 validation against this build, then:"
log_info "  sudo ./llama-cpp-promote.sh ${BUILD_TAG}"