unsloth/.github/workflows/docker-publish.yml
Daniel Han 3d563794af docker ci: aggressive runner disk reclaim before image builds
A staging run of the studio image build died with ENOSPC during the
Studio venv install: the hosted runners' default free space does not
fit the base image plus buildkit state plus the Studio layer. Drop all
unused preinstalled toolchains and the runner's preloaded docker
images in both build jobs.
2026-06-12 06:11:15 +00:00

484 lines
21 KiB
YAML

# Builds and publishes the Blackwell-compatible Unsloth Docker image.
#
# The build runs on free GitHub-hosted Ubuntu runners with NO GPU attached.
# This is possible because:
# 1. cu128 PyTorch wheels are fat binaries -- they already ship sm_70 through
# sm_120 SASS on amd64 (and sm_80;90;100;120 on aarch64), cross-compiled
# upstream by the PyTorch team.
# 2. The Dockerfile pins explicit wheel URLs (no --torch-backend=auto, no
# install.sh that introspects the host driver).
# 3. The build-time sanity check uses torch._C._cuda_getArchFlags(), which
# reads compiled wheel metadata and does NOT require a CUDA device.
# 4. UNSLOTH_COMPILE_DISABLE=1 prevents Unsloth from JIT-compiling a Triton
# kernel cache keyed to the (non-existent) build-host GPU.
#
# Multi-arch: build amd64 and arm64 in parallel on NATIVE GitHub runners
# (`ubuntu-latest` and `ubuntu-24.04-arm`, both free on public repos since
# Aug-2025), then merge the per-arch digests into a single multi-platform
# manifest. Native arm64 is ~3x faster than building aarch64 under QEMU,
# and avoids QEMU's occasional flakiness on long-running cu* installs.
# End users on DGX Spark / Grace pull the arm64 child natively; CUDA works
# as normal (no runtime emulation).
#
# Required repository secrets:
# DOCKERHUB_USERNAME, DOCKERHUB_TOKEN
#
# Optional repository variable (gates the smoke-test job):
# HAS_GPU_RUNNER = 'true' if a self-hosted GPU runner is available
name: Publish Blackwell Docker image
on:
push:
branches: [main]
tags: ['v*']
schedule:
- cron: '17 4 * * 1' # weekly Mon 04:17 UTC (off-the-hour on purpose)
workflow_dispatch:
inputs:
unsloth_ref:
description: 'unsloth git ref to bake in'
required: false
default: 'main'
unsloth_zoo_ref:
description: 'unsloth-zoo git ref to bake in'
required: false
default: 'main'
env:
REGISTRY: docker.io
IMAGE_NAME: unsloth/unsloth
# Serialise per-ref runs so two pushes to main (or two scheduled
# fires racing a manual dispatch) don't both retag `:latest` from
# different commits. Don't cancel in-progress runs -- the build is
# expensive and a half-built image left around in Docker Hub is
# worse than a slightly stale `:latest` for a few minutes.
concurrency:
group: docker-publish-${{ github.ref }}
cancel-in-progress: false
jobs:
# ---------------------------------------------------------------------------
# Per-arch build. The matrix fans out two parallel jobs on the matching
# native runner. Each pushes a single-arch image *by digest* (no human-
# readable tag), and the merge job below stitches the two digests into one
# multi-arch manifest under the real tags. This is the canonical pattern
# from docker/build-push-action's docs and avoids the "last push wins" race
# that you get when two jobs push the same tag separately.
# ---------------------------------------------------------------------------
build:
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
- platform: linux/arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
timeout-minutes: 90
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
# Free up ~20GB on the runner so cu128 wheels + cudnn fit. Layout is
# similar between the amd64 and arm64 runners but not identical -- the
# arm64 image lacks /usr/share/dotnet, hence `|| true`.
- name: Reclaim disk
run: |
# The hosted runners keep ~14-20 GB free, which is not enough for
# the image plus buildkit state (empirically confirmed: the Studio
# layer install died with ENOSPC on a staging run before this list
# was extended). None of these preinstalled toolchains are used
# here; some paths differ between the amd64 and arm64 runner
# images, hence `|| true`.
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
/opt/hostedtoolcache "$AGENT_TOOLSDIRECTORY" \
/usr/local/.ghcup /usr/share/swift \
/usr/local/share/powershell /usr/local/lib/node_modules \
/usr/local/julia* /opt/microsoft /usr/share/miniconda \
/opt/az /usr/local/share/boost /usr/local/share/chromium || true
sudo docker image prune -af >/dev/null 2>&1 || true
df -h /
- uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
# Pull the image label/annotation set we'll attach to the FINAL manifest.
# We don't apply tags at this layer because each per-arch build pushes by
# digest only; tags get attached by the merge job.
- name: Resolve labels
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
# Mirror the unsloth tag into the zoo ONLY when that tag actually
# exists there. unsloth's v* tags are Studio releases the zoo never
# cuts (the zoo repo currently has no tags at all), so blindly
# mirroring github.ref_name made every tag publish fail inside the
# Dockerfile's zoo install.
- name: Resolve unsloth-zoo ref
id: zoo_ref
run: |
REF="${{ github.event.inputs.unsloth_zoo_ref }}"
if [ -z "$REF" ] && [ "${{ startsWith(github.ref, 'refs/tags/') }}" = "true" ]; then
if git ls-remote --exit-code --tags https://github.com/unslothai/unsloth-zoo \
"refs/tags/${{ github.ref_name }}" >/dev/null 2>&1; then
REF="${{ github.ref_name }}"
fi
fi
echo "ref=${REF:-main}" >> "$GITHUB_OUTPUT"
echo "unsloth-zoo ref: ${REF:-main}"
- name: Build and push (per-arch by digest)
id: build
uses: docker/build-push-action@v6
with:
context: ./docker
file: ./docker/Dockerfile
platforms: ${{ matrix.platform }}
labels: ${{ steps.meta.outputs.labels }}
# Per-arch build cache. Keying on the platform suffix lets the two
# matrix legs reuse their own caches without colliding.
cache-from: type=gha,scope=build-${{ matrix.platform }}
cache-to: type=gha,scope=build-${{ matrix.platform }},mode=max
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
build-args: |
CUDA_VERSION=12.8.1
UBUNTU_VERSION=24.04
PYTHON_VERSION=3.12
# Workflow-dispatch: honour the explicit input. Tag pushes:
# bake the tag's source ref (e.g. v1.2.3) so the published
# tag image actually contains that release. Branch pushes and
# scheduled runs: bake the triggering commit SHA. Falls back
# to `main` for any other event class.
UNSLOTH_REF=${{ github.event.inputs.unsloth_ref || (startsWith(github.ref, 'refs/tags/') && github.ref_name) || github.sha || 'main' }}
# UNSLOTH_ZOO_REF comes from the resolve step above: explicit
# workflow-dispatch input, else the pushed tag IF the zoo repo
# has it, else `main`. SHA-based branch pushes always fall to
# `main` -- the SHA doesn't exist in the zoo repo.
UNSLOTH_ZOO_REF=${{ steps.zoo_ref.outputs.ref }}
# Stash the per-arch digest as an artifact for the merge job to pick up.
# Filenames need to be unique across the matrix; `platform` contains a
# slash so substitute it for a dash.
- name: Export digest
run: |
mkdir -p /tmp/digests
digest='${{ steps.build.outputs.digest }}'
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-base-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
# ---------------------------------------------------------------------------
# Merge the two per-arch digests into a multi-platform manifest under the
# real, user-facing tag(s). This job runs only after both `build` matrix
# legs finish successfully.
# ---------------------------------------------------------------------------
merge:
runs-on: ubuntu-latest
needs: build
timeout-minutes: 15
permissions:
contents: read
packages: write
outputs:
# Multi-arch manifest digest of the just-published base image. The
# build-studio job FROMs this exact digest so the Studio image always
# layers on the bits published by THIS run, not whatever `base`
# happens to point at when the job is scheduled.
digest: ${{ steps.manifest_digest.outputs.digest }}
steps:
- uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digests-base-*
merge-multiple: true
- uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Resolve tags
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
# The lean training image publishes under the base- prefix; the
# full Studio image (build-studio/merge-studio below) owns
# :latest, matching what the previous production image shipped.
# Only tag :base when the workflow ran on the default branch
# AND the operator did NOT override unsloth_ref on dispatch.
# Without the second condition a maintainer testing a feature
# SHA from main could overwrite :base with non-main source.
type=raw,value=base,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event.inputs.unsloth_ref == '' }}
type=ref,event=tag,prefix=base-
type=schedule,pattern=base-nightly
type=sha,prefix=base-sha-,format=short
- name: Create multi-arch manifest
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<<"$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
- name: Inspect the result
run: |
for tag in $(jq -r '.tags[]' <<<"$DOCKER_METADATA_OUTPUT_JSON"); do
echo "=== $tag ==="
docker buildx imagetools inspect "$tag"
done
- name: Export manifest digest
id: manifest_digest
run: |
TAG="$(jq -r '.tags[0]' <<<"$DOCKER_METADATA_OUTPUT_JSON")"
DIGEST="$(docker buildx imagetools inspect "$TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
test -n "$DIGEST"
echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
echo "base manifest: ${TAG} @ ${DIGEST}"
# ---------------------------------------------------------------------------
# Full image: base + Unsloth Studio + JupyterLab + sshd (Dockerfile.studio).
# This is what :latest points at, matching the service set of the previous
# production image. Same by-digest build + manifest-merge pattern as the
# base. FROMs the exact base manifest digest published by the merge job.
# The arm64 leg builds Studio's vite frontend natively on the arm runner;
# that is the long pole, hence the larger timeout.
# ---------------------------------------------------------------------------
build-studio:
needs: merge
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
- platform: linux/arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
timeout-minutes: 150
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@v4
- name: Reclaim disk
run: |
# The hosted runners keep ~14-20 GB free, which is not enough for
# the image plus buildkit state (empirically confirmed: the Studio
# layer install died with ENOSPC on a staging run before this list
# was extended). None of these preinstalled toolchains are used
# here; some paths differ between the amd64 and arm64 runner
# images, hence `|| true`.
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
/opt/hostedtoolcache "$AGENT_TOOLSDIRECTORY" \
/usr/local/.ghcup /usr/share/swift \
/usr/local/share/powershell /usr/local/lib/node_modules \
/usr/local/julia* /opt/microsoft /usr/share/miniconda \
/opt/az /usr/local/share/boost /usr/local/share/chromium || true
sudo docker image prune -af >/dev/null 2>&1 || true
df -h /
- uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Resolve labels
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
- name: Build and push (per-arch by digest)
id: build
uses: docker/build-push-action@v6
with:
context: ./docker
file: ./docker/Dockerfile.studio
platforms: ${{ matrix.platform }}
labels: ${{ steps.meta.outputs.labels }}
# mode=min (final layers only): a mode=max cache of this ~24GB
# image would blow straight through the 10GB per-repo GHA cache
# quota and evict the base build's cache for zero hit-rate gain.
cache-from: type=gha,scope=studio-${{ matrix.platform }}
cache-to: type=gha,scope=studio-${{ matrix.platform }},mode=min
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
build-args: |
BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.merge.outputs.digest }}
# Mirror of the base job's UNSLOTH_REF resolution so the Studio
# tree matches the unsloth baked into the base venv.
UNSLOTH_STUDIO_REF=${{ github.event.inputs.unsloth_ref || (startsWith(github.ref, 'refs/tags/') && github.ref_name) || github.sha || 'main' }}
- name: Export digest
run: |
mkdir -p /tmp/digests
digest='${{ steps.build.outputs.digest }}'
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-studio-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
merge-studio:
runs-on: ubuntu-latest
needs: build-studio
timeout-minutes: 15
permissions:
contents: read
packages: write
steps:
- uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digests-studio-*
merge-multiple: true
- uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Resolve tags
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
# The full Studio image owns the unprefixed namespace, headed by
# :latest. Same :latest gating rationale as the base job.
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event.inputs.unsloth_ref == '' }}
type=ref,event=tag
type=schedule,pattern=nightly
type=sha,prefix=sha-,format=short
- name: Create multi-arch manifest
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<<"$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
- name: Inspect the result
run: |
for tag in $(jq -r '.tags[]' <<<"$DOCKER_METADATA_OUTPUT_JSON"); do
echo "=== $tag ==="
docker buildx imagetools inspect "$tag"
done
# ---------------------------------------------------------------------------
# Optional: pull the freshly published image onto a self-hosted GPU runner
# and run smoke_test.py. Skipped automatically when no GPU runner is
# registered. Architecture matches whatever the runner is.
# ---------------------------------------------------------------------------
smoke-test:
needs: [merge, merge-studio]
if: ${{ vars.HAS_GPU_RUNNER == 'true' }}
runs-on: [self-hosted, gpu]
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
# Re-compute the tag list deterministically from the same metadata-action
# config the merge job used, so tag/schedule/SHA runs pull the image
# they just published instead of an unrelated tag from a prior run.
# IMPORTANT: keep the `enable=` expressions byte-identical to the
# corresponding merge jobs' gates above. The two used to differ
# (merge: ref + unsloth_ref guard; smoke: is_default_branch only),
# which meant workflow_dispatch with unsloth_ref defaulting to "main"
# would skip :latest on merge but still emit :latest as tags[0] on
# smoke -- so docker pull would fetch a previously-published :latest
# from Docker Hub, not the image just merged.
- name: Resolve published base tag
id: meta_base
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=raw,value=base,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event.inputs.unsloth_ref == '' }}
type=ref,event=tag,prefix=base-
type=schedule,pattern=base-nightly
type=sha,prefix=base-sha-,format=short
- name: Pull and smoke-test the base image
run: |
# Use the first tag from the metadata output -- that is the image we
# just published. Falls back to :base only when the metadata is
# empty (defensive; should not happen on default-branch runs).
TAG="$(jq -r '.tags[0] // ""' <<<"$STEPS_META_BASE_JSON")"
if [ -z "$TAG" ]; then
TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:base"
fi
echo "smoke-testing $TAG"
docker pull "$TAG"
docker run --rm --gpus all "$TAG" python /workspace/smoke_test.py
env:
STEPS_META_BASE_JSON: ${{ steps.meta_base.outputs.json }}
- name: Resolve published studio tag
id: meta_studio
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event.inputs.unsloth_ref == '' }}
type=ref,event=tag
type=schedule,pattern=nightly
type=sha,prefix=sha-,format=short
- name: Boot the full image and probe Studio + Jupyter
run: |
TAG="$(jq -r '.tags[0] // ""' <<<"$STEPS_META_STUDIO_JSON")"
if [ -z "$TAG" ]; then
TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest"
fi
echo "booting $TAG"
docker pull "$TAG"
CID="$(docker run -d --gpus all -p 18000:8000 -p 18888:8888 "$TAG")"
trap 'docker logs --tail 100 "$CID"; docker rm -f "$CID"' EXIT
ok_studio=0; ok_jupyter=0
for i in $(seq 1 60); do
if curl -fsS http://localhost:18000/api/health >/dev/null 2>&1; then ok_studio=1; fi
if curl -fsS http://localhost:18888/api >/dev/null 2>&1; then ok_jupyter=1; fi
[ "$ok_studio" = 1 ] && [ "$ok_jupyter" = 1 ] && break
sleep 5
done
[ "$ok_studio" = 1 ] || { echo "Studio /api/health never went healthy"; exit 1; }
[ "$ok_jupyter" = 1 ] || { echo "Jupyter /api never responded"; exit 1; }
echo "Studio + Jupyter healthy"
env:
STEPS_META_STUDIO_JSON: ${{ steps.meta_studio.outputs.json }}