docker: drop the local dev harnesses from the image PR
Remove seven dev-only scripts that never reach the image or CI: the .dockerignore whitelist excludes them from the build context, docker-publish.yml runs smoke_test.py via buildx with native arm64 runners (no QEMU setup script), and nothing else references them beyond a few comments. test_locally.sh, docker_confirm.sh/.ps1, setup_qemu.sh, hf_pull.sh, hf_push.sh and freeze.sh can return in a follow-up dev-tooling PR; this PR stays the image itself. Cuts 1105 lines and 7 files from the diff.
This commit is contained in:
Daniel Han
parent
4cfc63e74f
commit
da0e908d55
7 changed files with 0 additions and 1105 deletions
|
|
@ -1,233 +0,0 @@
|
|||
# docker_confirm.ps1 (Unsloth Docker image confirmation - Windows)
|
||||
# Confirms the published Unsloth Docker images actually work on this machine
|
||||
# through Docker Desktop: pulls them, checks WSL2 GPU passthrough (or CPU
|
||||
# fallback), runs a real 5-step LoRA training smoke, checks the baked
|
||||
# llama.cpp GGUF tooling, boots the full image and probes Studio +
|
||||
# JupyterLab, then prints a PASS/FAIL report.
|
||||
#
|
||||
# One-liner (PowerShell):
|
||||
# irm https://raw.githubusercontent.com/unslothai/unsloth/main/docker/docker_confirm.ps1 | iex
|
||||
#
|
||||
# What to expect per machine class:
|
||||
# Windows + NVIDIA (RTX 5070 / DGX Spark): GPU mode when Docker Desktop
|
||||
# uses the WSL2 backend with GPU support enabled (Settings > Resources).
|
||||
# Windows + AMD (Strix Halo): CPU mode - Docker Desktop has no ROCm
|
||||
# passthrough; training phases are skipped, Studio chat / Jupyter / GGUF
|
||||
# tooling still validate. Use the native install for AMD GPU work.
|
||||
#
|
||||
# Env overrides: $env:IMAGE, $env:BASE_IMAGE, $env:GPUS ('auto'|'all'|'none'),
|
||||
# $env:PORT_STUDIO (18000), $env:PORT_JUPYTER (18888), $env:WORK,
|
||||
# $env:SKIP_PULL, $env:SKIP_TRAIN, $env:KEEP
|
||||
|
||||
$ErrorActionPreference = "Continue"
|
||||
$IMAGE = if ($env:IMAGE) { $env:IMAGE } else { "unsloth/unsloth:latest" }
|
||||
$BASE_IMAGE = if ($env:BASE_IMAGE) { $env:BASE_IMAGE } else { "unsloth/unsloth:core" }
|
||||
$GPUS = if ($env:GPUS) { $env:GPUS } else { "auto" }
|
||||
$PORT_STUDIO = if ($env:PORT_STUDIO) { $env:PORT_STUDIO } else { 18000 }
|
||||
$PORT_JUPYTER = if ($env:PORT_JUPYTER) { $env:PORT_JUPYTER } else { 18888 }
|
||||
$WORK = if ($env:WORK) { $env:WORK } else { Join-Path $HOME "unsloth_docker_test" }
|
||||
$SKIP_PULL = $env:SKIP_PULL -eq "1"
|
||||
$SKIP_TRAIN = $env:SKIP_TRAIN -eq "1"
|
||||
$KEEP = $env:KEEP -eq "1"
|
||||
|
||||
$script:PASS_N = 0; $script:FAIL_N = 0; $script:WARN_N = 0; $script:STUDIO_CID = ""
|
||||
function Bold($m){ Write-Host $m -ForegroundColor White }
|
||||
function Ok($m) { Write-Host " [PASS] $m" -ForegroundColor Green; $script:PASS_N++ }
|
||||
function Bad($m) { Write-Host " [FAIL] $m" -ForegroundColor Red; $script:FAIL_N++ }
|
||||
function Warn($m){ Write-Host " [WARN] $m" -ForegroundColor Yellow; $script:WARN_N++ }
|
||||
function Info($m){ Write-Host " $m" }
|
||||
function Hr() { Write-Host ("-" * 63) }
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $WORK | Out-Null
|
||||
Write-Host ""; Bold "=== Unsloth Docker image confirmation (Windows) ==="
|
||||
Write-Host "scratch dir : $WORK"; Hr
|
||||
|
||||
# 1) Host detection -----------------------------------------------------------
|
||||
Bold "1) Host detection"
|
||||
Info ("windows : " + [System.Environment]::OSVersion.VersionString + " " + $env:PROCESSOR_ARCHITECTURE)
|
||||
if (-not (Get-Command docker -ErrorAction SilentlyContinue)) {
|
||||
Bad "docker not found - install Docker Desktop first"
|
||||
Bold "RESULT: cannot continue without docker."; exit 1
|
||||
}
|
||||
docker info *> $null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
Bad "docker daemon not reachable - start Docker Desktop"
|
||||
Bold "RESULT: cannot continue without a reachable docker daemon."; exit 1
|
||||
}
|
||||
Ok ("docker daemon reachable (" + (docker --version) + ")")
|
||||
$osType = (docker info --format "{{.OSType}}" 2>$null)
|
||||
if ($osType -ne "linux") {
|
||||
Bad "Docker Desktop is in Windows-container mode (OSType=$osType) - switch to Linux containers"
|
||||
}
|
||||
|
||||
$GPU_MODE = $false
|
||||
if ($GPUS -eq "none") {
|
||||
Info "GPU mode : disabled by GPUS=none"
|
||||
} elseif (Get-Command nvidia-smi -ErrorAction SilentlyContinue) {
|
||||
$gpus = nvidia-smi --query-gpu=index,name,compute_cap --format=csv,noheader 2>$null
|
||||
if ($LASTEXITCODE -eq 0 -and $gpus) {
|
||||
$gpus | ForEach-Object { Info (" - " + $_) }
|
||||
Ok "NVIDIA GPU visible on the host - probing WSL2 passthrough below"
|
||||
$GPU_MODE = $true
|
||||
} else {
|
||||
Info "nvidia-smi present but no GPU listed"
|
||||
}
|
||||
} else {
|
||||
Info "no NVIDIA GPU on the host (or nvidia-smi missing)"
|
||||
}
|
||||
if (-not $GPU_MODE) {
|
||||
Warn "CPU mode: training phases are skipped; Studio chat / Jupyter / GGUF tooling still validate"
|
||||
}
|
||||
Hr
|
||||
|
||||
# 2) Pull images --------------------------------------------------------------
|
||||
Bold "2) Pull images"
|
||||
foreach ($img in @($BASE_IMAGE, $IMAGE)) {
|
||||
if ($SKIP_PULL) {
|
||||
docker image inspect $img *> $null
|
||||
if ($LASTEXITCODE -eq 0) { Ok "local image present: $img" } else { Bad "SKIP_PULL=1 but image missing locally: $img" }
|
||||
} else {
|
||||
$log = Join-Path $WORK ("pull_" + ($img -replace "[/:]", "_") + ".log")
|
||||
docker pull $img *> $log
|
||||
if ($LASTEXITCODE -eq 0) { Ok "pulled $img" }
|
||||
else {
|
||||
docker image inspect $img *> $null
|
||||
# Locally built tags are not on a registry; presence is what matters.
|
||||
if ($LASTEXITCODE -eq 0) { Warn "not pullable but present locally: $img" }
|
||||
else { Bad "could not pull $img (see $log)" }
|
||||
}
|
||||
}
|
||||
}
|
||||
Hr
|
||||
|
||||
# 3) Container runtime check --------------------------------------------------
|
||||
Bold "3) Container runtime check"
|
||||
# Mirror docker_confirm.sh's GPU selector translation: bare indices and
|
||||
# comma lists become device= selectors (Docker reads a bare integer for
|
||||
# --gpus as a COUNT, not an index). Built as an args array so every docker
|
||||
# run call splats it identically.
|
||||
#
|
||||
# Comma lists are special: docker CSV-parses the --gpus value, so a list
|
||||
# must arrive as a literal "device=0,1" INCLUDING the double quotes. How
|
||||
# PowerShell passes embedded quotes to native commands changed in 7.3
|
||||
# (PSNativeCommandArgumentPassing), so pick the escaping per version;
|
||||
# single selectors need no quoting anywhere.
|
||||
$GPU_SELECTOR = "all"
|
||||
if ($GPUS -notin @("auto", "all", "none")) {
|
||||
$sel = $GPUS -replace "^device=", ""
|
||||
if ($sel -match ",") {
|
||||
if ($PSVersionTable.PSVersion -ge [version]"7.3") { $GPU_SELECTOR = '"device=' + $sel + '"' }
|
||||
else { $GPU_SELECTOR = '\"device=' + $sel + '\"' }
|
||||
} else {
|
||||
$GPU_SELECTOR = "device=$sel"
|
||||
}
|
||||
}
|
||||
$GpuRunArgs = @("--gpus", $GPU_SELECTOR)
|
||||
if ($GPU_MODE) {
|
||||
$log = Join-Path $WORK "gpu_check.log"
|
||||
docker run --rm @GpuRunArgs $BASE_IMAGE python -c "import torch; assert torch.cuda.is_available(); print('torch', torch.__version__, '-', torch.cuda.get_device_name(0))" *> $log
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Ok ("torch.cuda available in-container: " + (Get-Content $log -Tail 1))
|
||||
} else {
|
||||
Bad "GPU passthrough failed (see $log) - check Docker Desktop WSL2 GPU support; falling back to CPU mode"
|
||||
Get-Content $log -Tail 5 | ForEach-Object { Info $_ }
|
||||
$GPU_MODE = $false
|
||||
}
|
||||
}
|
||||
if (-not $GPU_MODE) {
|
||||
$log = Join-Path $WORK "cpu_check.log"
|
||||
docker run --rm -e UNSLOTH_ALLOW_CPU=1 $BASE_IMAGE python -c "import torch; print('torch', torch.__version__, 'cpu-mode ok')" *> $log
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Ok ("CPU mode boots: " + (Get-Content $log -Tail 1))
|
||||
} else {
|
||||
Bad "container failed to start even in CPU mode (see $log)"
|
||||
Get-Content $log -Tail 5 | ForEach-Object { Info $_ }
|
||||
}
|
||||
}
|
||||
Hr
|
||||
|
||||
# 4) Training smoke (GPU only) ------------------------------------------------
|
||||
Bold "4) Training smoke"
|
||||
if ($GPU_MODE -and -not $SKIP_TRAIN) {
|
||||
$log = Join-Path $WORK "train_smoke.log"
|
||||
$hfArgs = @(); if ($env:HF_TOKEN) { $hfArgs = @("-e", "HF_TOKEN") }
|
||||
docker run --rm @GpuRunArgs --ipc=host @hfArgs $BASE_IMAGE python /workspace/smoke_test.py *> $log
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Ok "smoke_test.py: 5 LoRA steps completed"
|
||||
Select-String -Path $log -Pattern "^step|loss" | Select-Object -Last 5 | ForEach-Object { Info $_.Line }
|
||||
} else {
|
||||
Bad "training smoke failed (see $log)"
|
||||
Get-Content $log -Tail 10 | ForEach-Object { Info $_ }
|
||||
}
|
||||
} else {
|
||||
Warn "skipped (CPU mode or SKIP_TRAIN=1)"
|
||||
}
|
||||
Hr
|
||||
|
||||
# 5) GGUF tooling -------------------------------------------------------------
|
||||
Bold "5) GGUF tooling (baked llama.cpp)"
|
||||
$log = Join-Path $WORK "gguf_check.log"
|
||||
docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 $BASE_IMAGE bash -c 'set -e; test -x "$UNSLOTH_LLAMA_CPP_PATH/llama-quantize"; test -f "$UNSLOTH_LLAMA_CPP_PATH/convert_hf_to_gguf.py"; "$UNSLOTH_LLAMA_CPP_PATH/llama-server" --version 2>&1 | head -2' *> $log
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Ok "llama-quantize + llama-server + convert_hf_to_gguf.py present and runnable"
|
||||
Select-String -Path $log -Pattern "version" | Select-Object -First 2 | ForEach-Object { Info $_.Line }
|
||||
} else {
|
||||
Bad "baked llama.cpp check failed (see $log)"
|
||||
Get-Content $log -Tail 5 | ForEach-Object { Info $_ }
|
||||
}
|
||||
Hr
|
||||
|
||||
# 5b) vLLM (GRPO fast_inference=True) -----------------------------------------
|
||||
Bold "5b) vLLM (GRPO fast_inference=True)"
|
||||
$log = Join-Path $WORK "vllm_check.log"
|
||||
docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 $BASE_IMAGE python -c 'import vllm; print("vllm", vllm.__version__)' *> $log
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
Ok ("vllm importable: " + (Get-Content $log -Tail 1))
|
||||
} else {
|
||||
$imgArch = docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 $BASE_IMAGE uname -m 2>$null
|
||||
if ($imgArch -eq "x86_64") {
|
||||
Bad "vllm missing or broken on x86_64 image (see $log)"
|
||||
Get-Content $log -Tail 3 | ForEach-Object { Info $_ }
|
||||
} else {
|
||||
Warn "vllm not available on $imgArch image; GRPO fast_inference=True unavailable (arm64 wheels are newer, fail-soft at image build)"
|
||||
}
|
||||
}
|
||||
Hr
|
||||
|
||||
# 6) Studio + JupyterLab ------------------------------------------------------
|
||||
Bold "6) Studio + JupyterLab (full image)"
|
||||
$runArgs = @("-d", "-p", "${PORT_STUDIO}:8000", "-p", "${PORT_JUPYTER}:8888")
|
||||
if ($GPU_MODE) { $runArgs += $GpuRunArgs } else { $runArgs += @("-e", "UNSLOTH_ALLOW_CPU=1") }
|
||||
$script:STUDIO_CID = (docker run @runArgs $IMAGE 2>(Join-Path $WORK "studio_run.err"))
|
||||
if (-not $script:STUDIO_CID) {
|
||||
Bad ("full image failed to start (see " + (Join-Path $WORK "studio_run.err") + ")")
|
||||
} else {
|
||||
Info ("container : " + $script:STUDIO_CID.Substring(0, 12) + " (studio http://localhost:$PORT_STUDIO, jupyter http://localhost:$PORT_JUPYTER)")
|
||||
$okStudio = $false; $okJupyter = $false
|
||||
foreach ($i in 1..60) {
|
||||
if (-not $okStudio) { try { Invoke-WebRequest -UseBasicParsing -Uri "http://localhost:$PORT_STUDIO/api/health" -TimeoutSec 4 | Out-Null; $okStudio = $true } catch {} }
|
||||
# /login, not /api: a password hash is always configured so /api returns 403.
|
||||
if (-not $okJupyter) { try { Invoke-WebRequest -UseBasicParsing -Uri "http://localhost:$PORT_JUPYTER/login" -TimeoutSec 4 | Out-Null; $okJupyter = $true } catch {} }
|
||||
if ($okStudio -and $okJupyter) { break }
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
if ($okStudio) { Ok "Studio /api/health healthy" } else { Bad "Studio /api/health never went healthy (docker logs $($script:STUDIO_CID.Substring(0,12)))"; docker logs --tail 15 $script:STUDIO_CID 2>&1 | ForEach-Object { Info $_ } }
|
||||
if ($okJupyter) { Ok "JupyterLab /login responding" } else { Bad "JupyterLab /login never responded" }
|
||||
}
|
||||
Hr
|
||||
|
||||
# Summary ---------------------------------------------------------------------
|
||||
Bold "=== SUMMARY ==="
|
||||
Write-Host "images : $IMAGE / $BASE_IMAGE"
|
||||
Write-Host ("gpu_mode : " + $GPU_MODE)
|
||||
Write-Host "logs : $WORK"
|
||||
Write-Host "PASS: $script:PASS_N WARN: $script:WARN_N FAIL: $script:FAIL_N"
|
||||
if (-not $KEEP -and $script:STUDIO_CID) { docker rm -f $script:STUDIO_CID *> $null }
|
||||
elseif ($KEEP -and $script:STUDIO_CID) { Write-Host ("container " + $script:STUDIO_CID.Substring(0,12) + " left running (KEEP=1): studio :$PORT_STUDIO jupyter :$PORT_JUPYTER") }
|
||||
if ($script:FAIL_N -eq 0) {
|
||||
Bold "RESULT: CONFIRMED - the Unsloth Docker images work on this machine."
|
||||
exit 0
|
||||
} else {
|
||||
Bold "RESULT: $script:FAIL_N hard failure(s) - paste this whole output back."
|
||||
exit 1
|
||||
}
|
||||
|
|
@ -1,266 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# docker_confirm.sh (Unsloth Docker image confirmation - Linux / WSL2 / macOS)
|
||||
# Confirms the published Unsloth Docker images actually work on this machine:
|
||||
# pulls them, checks GPU passthrough (or CPU fallback), runs a real 5-step
|
||||
# LoRA training smoke, checks the baked llama.cpp GGUF tooling, boots the
|
||||
# full image and probes Studio + JupyterLab, then prints a PASS/FAIL report.
|
||||
#
|
||||
# Nothing is installed on the host beyond the Docker images themselves; the
|
||||
# containers it starts are removed afterwards (KEEP=1 keeps them running).
|
||||
#
|
||||
# One-liner:
|
||||
# curl -fsSL https://raw.githubusercontent.com/unslothai/unsloth/main/docker/docker_confirm.sh | bash
|
||||
#
|
||||
# What to expect per machine class:
|
||||
# Linux + NVIDIA (B200 / RTX 6000 / RTX 50-series). GPU mode, all phases.
|
||||
# Windows + NVIDIA via Docker Desktop (WSL2 backend): run inside the WSL2
|
||||
# distro or Git Bash. GPU mode if Docker Desktop has WSL2 GPU enabled.
|
||||
# DGX Spark / GB10 (Linux arm64): GPU mode, the arm64 image child is pulled
|
||||
# automatically.
|
||||
# macOS (M-series) and Windows + AMD (Strix Halo): CPU mode is auto-detected
|
||||
# (no NVIDIA passthrough exists for these); training phases are skipped,
|
||||
# Studio chat / Jupyter / GGUF tooling still validate.
|
||||
#
|
||||
# Env overrides: IMAGE (default unsloth/unsloth:latest)
|
||||
# BASE_IMAGE (default unsloth/unsloth:core)
|
||||
# GPUS=all|none|0|0,1 (default: auto-detect)
|
||||
# PORT_STUDIO=18000 PORT_JUPYTER=18888
|
||||
# WORK=~/unsloth_docker_test (logs)
|
||||
# HF_CACHE=~/.cache/huggingface (mounted to speed model pulls)
|
||||
# SKIP_PULL=1 (use local images) SKIP_TRAIN=1 KEEP=1
|
||||
#
|
||||
set -uo pipefail
|
||||
|
||||
IMAGE="${IMAGE:-unsloth/unsloth:latest}"
|
||||
BASE_IMAGE="${BASE_IMAGE:-unsloth/unsloth:core}"
|
||||
GPUS="${GPUS:-auto}"
|
||||
PORT_STUDIO="${PORT_STUDIO:-18000}"
|
||||
PORT_JUPYTER="${PORT_JUPYTER:-18888}"
|
||||
WORK="${WORK:-$HOME/unsloth_docker_test}"
|
||||
HF_CACHE="${HF_CACHE:-$HOME/.cache/huggingface}"
|
||||
SKIP_PULL="${SKIP_PULL:-0}"
|
||||
SKIP_TRAIN="${SKIP_TRAIN:-0}"
|
||||
KEEP="${KEEP:-0}"
|
||||
ARCH="$(uname -m)"
|
||||
OS="$(uname -s)"
|
||||
|
||||
PASS_N=0; FAIL_N=0; WARN_N=0; STUDIO_CID=""
|
||||
bold(){ printf '\033[1m%s\033[0m\n' "$*"; }
|
||||
ok(){ printf ' [PASS] %s\n' "$*"; PASS_N=$((PASS_N+1)); }
|
||||
bad(){ printf ' [FAIL] %s\n' "$*"; FAIL_N=$((FAIL_N+1)); }
|
||||
warn(){ printf ' [WARN] %s\n' "$*"; WARN_N=$((WARN_N+1)); }
|
||||
info(){ printf ' %s\n' "$*"; }
|
||||
hr(){ printf -- '---------------------------------------------------------------\n'; }
|
||||
|
||||
cleanup(){
|
||||
if [ "$KEEP" != "1" ] && [ -n "$STUDIO_CID" ]; then
|
||||
docker rm -f "$STUDIO_CID" >/dev/null 2>&1
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir -p "$WORK" "$HF_CACHE"
|
||||
echo; bold "=== Unsloth Docker image confirmation ==="
|
||||
echo "scratch dir : $WORK"; hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 1. Host detection
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "1) Host detection"
|
||||
info "uname : $OS $ARCH ($(uname -r 2>/dev/null))"
|
||||
IS_WSL=0
|
||||
grep -qiE "microsoft|wsl" /proc/version 2>/dev/null && { IS_WSL=1; info "WSL : yes"; }
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
bad "docker not found on PATH - install Docker Engine / Docker Desktop first"
|
||||
echo; bold "RESULT: cannot continue without docker."; exit 1
|
||||
fi
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
bad "docker daemon not reachable (permission denied or not running)"
|
||||
info "try: sudo usermod -aG docker \$USER && re-login, or start Docker Desktop"
|
||||
echo; bold "RESULT: cannot continue without a reachable docker daemon."; exit 1
|
||||
fi
|
||||
ok "docker daemon reachable ($(docker --version 2>/dev/null))"
|
||||
|
||||
GPU_MODE=0
|
||||
NVRT_LISTED=0
|
||||
if [ "$GPUS" = "none" ]; then
|
||||
info "GPU mode : disabled by GPUS=none"
|
||||
elif command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q '^GPU'; then
|
||||
info "GPU(s) :"
|
||||
nvidia-smi --query-gpu=index,name,compute_cap --format=csv,noheader 2>/dev/null | sed 's/^/ - /'
|
||||
# `docker info | grep Runtimes:.*nvidia` misses CDI setups (docker 25+
|
||||
# with nvidia-ctk cdi) and Docker Desktop's WSL2 backend, both of which
|
||||
# expose GPUs without a host-visible runtime entry. Treat the listing as
|
||||
# a hint only; phase 3 probes --gpus for real and demotes to CPU mode if
|
||||
# the probe fails.
|
||||
if docker info 2>/dev/null | grep -qi 'Runtimes:.*nvidia'; then
|
||||
ok "NVIDIA GPU visible and docker lists the nvidia runtime"
|
||||
NVRT_LISTED=1
|
||||
else
|
||||
warn "nvidia runtime not listed by docker info (normal under CDI or Docker Desktop WSL2) - probing --gpus directly in phase 3"
|
||||
fi
|
||||
GPU_MODE=1
|
||||
else
|
||||
info "no NVIDIA GPU on the host (or nvidia-smi missing)"
|
||||
fi
|
||||
if [ "$GPU_MODE" = "0" ]; then
|
||||
warn "CPU mode: training phases are skipped; Studio chat / Jupyter / GGUF tooling still validate"
|
||||
fi
|
||||
GPU_FLAG=(--gpus all)
|
||||
case "$GPUS" in
|
||||
auto|all|none) ;;
|
||||
*) GPU_FLAG=(--gpus "\"device=${GPUS}\"") ;;
|
||||
esac
|
||||
hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 2. Pull images
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "2) Pull images"
|
||||
for img in "$BASE_IMAGE" "$IMAGE"; do
|
||||
if [ "$SKIP_PULL" = "1" ]; then
|
||||
docker image inspect "$img" >/dev/null 2>&1 && ok "local image present: $img" || bad "SKIP_PULL=1 but image missing locally: $img"
|
||||
elif docker pull "$img" >"$WORK/pull_$(echo "$img" | tr '/:' '__').log" 2>&1; then
|
||||
ok "pulled $img"
|
||||
elif docker image inspect "$img" >/dev/null 2>&1; then
|
||||
# Locally built tags (test_locally.sh / docker build) are not on a
|
||||
# registry; that is fine as long as the image is present.
|
||||
warn "not pullable but present locally: $img"
|
||||
else
|
||||
bad "could not pull $img (see $WORK/pull_*.log)"
|
||||
fi
|
||||
done
|
||||
hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 3. GPU passthrough / CPU fallback inside the container
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "3) Container runtime check"
|
||||
if [ "$GPU_MODE" = "1" ]; then
|
||||
if docker run --rm "${GPU_FLAG[@]}" "$BASE_IMAGE" python -c \
|
||||
"import torch; assert torch.cuda.is_available(); print('torch', torch.__version__, '-', torch.cuda.get_device_name(0))" \
|
||||
>"$WORK/gpu_check.log" 2>&1; then
|
||||
ok "torch.cuda available in-container: $(tail -1 "$WORK/gpu_check.log")"
|
||||
else
|
||||
if [ "$NVRT_LISTED" = "1" ]; then
|
||||
bad "GPU passthrough failed despite a listed nvidia runtime (see $WORK/gpu_check.log) - falling back to CPU mode"
|
||||
else
|
||||
warn "--gpus probe failed - docker has no nvidia runtime or CDI spec (install nvidia-container-toolkit); falling back to CPU mode"
|
||||
fi
|
||||
tail -5 "$WORK/gpu_check.log" | sed 's/^/ /'
|
||||
GPU_MODE=0
|
||||
fi
|
||||
fi
|
||||
if [ "$GPU_MODE" = "0" ]; then
|
||||
if docker run --rm -e UNSLOTH_ALLOW_CPU=1 "$BASE_IMAGE" python -c \
|
||||
"import torch; print('torch', torch.__version__, 'cpu-mode ok')" \
|
||||
>"$WORK/cpu_check.log" 2>&1; then
|
||||
ok "CPU mode boots: $(tail -1 "$WORK/cpu_check.log")"
|
||||
else
|
||||
bad "container failed to start even in CPU mode (see $WORK/cpu_check.log)"
|
||||
tail -5 "$WORK/cpu_check.log" | sed 's/^/ /'
|
||||
fi
|
||||
fi
|
||||
hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 4. Training smoke (GPU only): 5 LoRA steps on Llama-3.2-1B 4-bit
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "4) Training smoke"
|
||||
if [ "$GPU_MODE" = "1" ] && [ "$SKIP_TRAIN" != "1" ]; then
|
||||
if docker run --rm "${GPU_FLAG[@]}" --ipc=host \
|
||||
-v "$HF_CACHE":/workspace/.cache/huggingface \
|
||||
${HF_TOKEN:+-e HF_TOKEN} \
|
||||
"$BASE_IMAGE" python /workspace/smoke_test.py >"$WORK/train_smoke.log" 2>&1; then
|
||||
ok "smoke_test.py: 5 LoRA steps completed"
|
||||
grep -E '^step|loss' "$WORK/train_smoke.log" | tail -5 | sed 's/^/ /'
|
||||
else
|
||||
bad "training smoke failed (see $WORK/train_smoke.log)"
|
||||
tail -10 "$WORK/train_smoke.log" | sed 's/^/ /'
|
||||
fi
|
||||
else
|
||||
warn "skipped (CPU mode or SKIP_TRAIN=1)"
|
||||
fi
|
||||
hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 5. GGUF tooling: baked llama.cpp prebuilt
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "5) GGUF tooling (baked llama.cpp)"
|
||||
if docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 "$BASE_IMAGE" bash -c '
|
||||
set -e
|
||||
test -x "$UNSLOTH_LLAMA_CPP_PATH/llama-quantize"
|
||||
test -f "$UNSLOTH_LLAMA_CPP_PATH/convert_hf_to_gguf.py"
|
||||
"$UNSLOTH_LLAMA_CPP_PATH/llama-server" --version 2>&1 | head -2
|
||||
cat "$UNSLOTH_LLAMA_CPP_PATH/UNSLOTH_PREBUILT_INFO.json" 2>/dev/null | head -5
|
||||
' >"$WORK/gguf_check.log" 2>&1; then
|
||||
ok "llama-quantize + llama-server + convert_hf_to_gguf.py present and runnable"
|
||||
grep -E 'version|asset' "$WORK/gguf_check.log" | head -3 | sed 's/^/ /'
|
||||
else
|
||||
bad "baked llama.cpp check failed (see $WORK/gguf_check.log)"
|
||||
tail -5 "$WORK/gguf_check.log" | sed 's/^/ /'
|
||||
fi
|
||||
hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 5b. vLLM (GRPO fast_inference=True)
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "5b) vLLM (GRPO fast_inference=True)"
|
||||
if docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 "$BASE_IMAGE" \
|
||||
python -c 'import vllm; print("vllm", vllm.__version__)' \
|
||||
>"$WORK/vllm_check.log" 2>&1; then
|
||||
ok "vllm importable: $(grep -oE 'vllm [0-9][^ ]*' "$WORK/vllm_check.log" | head -1)"
|
||||
else
|
||||
IMG_ARCH="$(docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 "$BASE_IMAGE" uname -m 2>/dev/null || echo unknown)"
|
||||
if [ "$IMG_ARCH" = "x86_64" ]; then
|
||||
bad "vllm missing or broken on x86_64 image (see $WORK/vllm_check.log)"
|
||||
tail -3 "$WORK/vllm_check.log" | sed 's/^/ /'
|
||||
else
|
||||
warn "vllm not available on $IMG_ARCH image; GRPO fast_inference=True unavailable (arm64 wheels are newer, fail-soft at image build)"
|
||||
fi
|
||||
fi
|
||||
hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# 6. Full image: Studio + JupyterLab boot
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "6) Studio + JupyterLab (full image)"
|
||||
RUN_ARGS=(-d -p "$PORT_STUDIO":8000 -p "$PORT_JUPYTER":8888)
|
||||
if [ "$GPU_MODE" = "1" ]; then RUN_ARGS+=("${GPU_FLAG[@]}"); else RUN_ARGS+=(-e UNSLOTH_ALLOW_CPU=1); fi
|
||||
STUDIO_CID="$(docker run "${RUN_ARGS[@]}" "$IMAGE" 2>"$WORK/studio_run.err")" || STUDIO_CID=""
|
||||
if [ -z "$STUDIO_CID" ]; then
|
||||
bad "full image failed to start (see $WORK/studio_run.err)"
|
||||
else
|
||||
info "container : ${STUDIO_CID:0:12} (studio http://localhost:$PORT_STUDIO, jupyter http://localhost:$PORT_JUPYTER)"
|
||||
ok_studio=0; ok_jupyter=0
|
||||
for _ in $(seq 1 60); do
|
||||
if [ "$ok_studio" = 0 ] && curl -fsS "http://localhost:$PORT_STUDIO/api/health" >/dev/null 2>&1; then ok_studio=1; fi
|
||||
# /login, not /api: a password hash is always configured so /api returns 403.
|
||||
if [ "$ok_jupyter" = 0 ] && curl -fsS "http://localhost:$PORT_JUPYTER/login" >/dev/null 2>&1; then ok_jupyter=1; fi
|
||||
[ "$ok_studio" = 1 ] && [ "$ok_jupyter" = 1 ] && break
|
||||
sleep 5
|
||||
done
|
||||
[ "$ok_studio" = 1 ] && ok "Studio /api/health healthy" || { bad "Studio /api/health never went healthy (docker logs ${STUDIO_CID:0:12})"; docker logs --tail 15 "$STUDIO_CID" 2>&1 | sed 's/^/ /'; }
|
||||
[ "$ok_jupyter" = 1 ] && ok "JupyterLab /login responding" || bad "JupyterLab /login never responded"
|
||||
fi
|
||||
hr
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Summary
|
||||
# --------------------------------------------------------------------------- #
|
||||
bold "=== SUMMARY ==="
|
||||
echo "host : $OS $ARCH wsl=$IS_WSL gpu_mode=$GPU_MODE"
|
||||
echo "images : $IMAGE / $BASE_IMAGE"
|
||||
echo "logs : $WORK"
|
||||
echo "PASS: $PASS_N WARN: $WARN_N FAIL: $FAIL_N"
|
||||
if [ "$KEEP" = "1" ] && [ -n "$STUDIO_CID" ]; then
|
||||
echo "container ${STUDIO_CID:0:12} left running (KEEP=1): studio :$PORT_STUDIO jupyter :$PORT_JUPYTER"
|
||||
fi
|
||||
if [ "$FAIL_N" -eq 0 ]; then
|
||||
bold "RESULT: CONFIRMED - the Unsloth Docker images work on this machine."
|
||||
exit 0
|
||||
else
|
||||
bold "RESULT: $FAIL_N hard failure(s) - paste this whole output back."
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Pull the lockfile out of a built image so the next rebuild can be byte-identical.
|
||||
#
|
||||
# ./freeze.sh # extracts to requirements.lock.txt next to Dockerfile
|
||||
# ./freeze.sh some-tag-or-digest # custom source
|
||||
#
|
||||
# To rebuild against the frozen lockfile later, replace the `pip install` lines
|
||||
# in the Dockerfile with `pip install -r /tmp/requirements.lock.txt --no-deps`
|
||||
# (mounted via `docker build --build-context lock=./requirements.lock.txt`).
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
SRC="${1:-unsloth-blackwell:latest}"
|
||||
DEST="${2:-./requirements.lock.txt}"
|
||||
|
||||
CID=$(docker create "${SRC}")
|
||||
trap 'docker rm -f "${CID}" >/dev/null' EXIT
|
||||
|
||||
docker cp "${CID}:/opt/unsloth-venv/requirements.lock.txt" "${DEST}"
|
||||
echo "Wrote ${DEST}"
|
||||
echo
|
||||
echo "Top of lockfile:"
|
||||
head -20 "${DEST}"
|
||||
echo
|
||||
echo "Lines: $(wc -l < "${DEST}")"
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Simulate `docker pull <image>` against a Hugging Face Hub model repo.
|
||||
#
|
||||
# Counterpart to docker/hf_push.sh -- downloads the tar.gz blob from the HF
|
||||
# repo and `docker load`s it.
|
||||
#
|
||||
# Usage:
|
||||
# bash docker/hf_pull.sh <hf_repo> [<blob>] [<verify_tag>]
|
||||
# bash docker/hf_pull.sh danielhanchen/unsloth-blackwell-docker unsloth-blackwell-test.tar.gz unsloth-blackwell:test
|
||||
#
|
||||
# Requires: docker, pigz (or gzip), hf (or huggingface-cli) authenticated
|
||||
# (read scope is sufficient for public repos: `hf auth login`).
|
||||
set -euo pipefail
|
||||
|
||||
REPO="${1:?usage: hf_pull.sh <hf_repo> [<blob>] [<verify_tag>]}"
|
||||
BLOB="${2:-unsloth-blackwell.tar.gz}"
|
||||
VERIFY="${3:-}"
|
||||
WORK="${HF_PULL_TMP:-/tmp}"
|
||||
|
||||
command -v docker >/dev/null || { echo "ERROR: docker not on PATH"; exit 1; }
|
||||
|
||||
# Prefer the new `hf` CLI. The old `huggingface-cli` was deprecated in
|
||||
# huggingface_hub >= 0.27 and silently exits with a deprecation notice
|
||||
# instead of doing the download, so we treat its presence as a fallback
|
||||
# only and warn if it's all we have.
|
||||
if command -v hf >/dev/null 2>&1; then
|
||||
HF_CMD=(hf download)
|
||||
elif command -v huggingface-cli >/dev/null 2>&1; then
|
||||
echo "WARN: 'hf' not found, falling back to 'huggingface-cli' (deprecated)" >&2
|
||||
HF_CMD=(huggingface-cli download)
|
||||
else
|
||||
echo "ERROR: need 'hf' (pip install -U huggingface_hub) or 'huggingface-cli'"; exit 1
|
||||
fi
|
||||
DECOMPRESSOR=$(command -v pigz || command -v gzip) || { echo "ERROR: need pigz or gzip"; exit 1; }
|
||||
|
||||
DEST="${WORK}/$(basename "${BLOB}")"
|
||||
echo ">> downloading ${REPO}/${BLOB} -> ${DEST} (via: ${HF_CMD[*]})"
|
||||
"${HF_CMD[@]}" "${REPO}" "${BLOB}" --repo-type=model --local-dir "${WORK}"
|
||||
test -s "${DEST}" || { echo "ERROR: download produced no file at ${DEST}"; exit 1; }
|
||||
ls -lh "${DEST}"
|
||||
|
||||
echo ">> loading into docker (using ${DECOMPRESSOR##*/})"
|
||||
"${DECOMPRESSOR}" -d -c "${DEST}" | docker load
|
||||
|
||||
if [[ -n "${VERIFY}" ]]; then
|
||||
if docker image inspect "${VERIFY}" >/dev/null 2>&1; then
|
||||
echo ">> verified: ${VERIFY} is loaded"
|
||||
docker image inspect --format 'image_id={{.Id}} size={{.Size}}' "${VERIFY}"
|
||||
else
|
||||
echo "WARN: expected tag ${VERIFY} not found after load. docker images:"
|
||||
docker images
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# Simulate `docker push <image>` against a Hugging Face Hub model repo.
|
||||
#
|
||||
# HF Hub doesn't act as an OCI registry for arbitrary images (only Spaces have
|
||||
# that). So we approximate the push by:
|
||||
# 1. docker save | pigz -> single tar.gz blob
|
||||
# 2. huggingface-cli upload to <repo>/{tag}.tar.gz
|
||||
#
|
||||
# This is good for cross-host testing where you want one canonical place to
|
||||
# pull from. For the real release, use Docker Hub or GHCR with `docker push`,
|
||||
# which gives you layer dedup, manifest negotiation, and standard `docker pull`
|
||||
# UX -- see .github/workflows/docker-publish.yml in this repo.
|
||||
#
|
||||
# Usage:
|
||||
# bash docker/hf_push.sh <image:tag> <hf_repo>
|
||||
# bash docker/hf_push.sh unsloth-blackwell:test danielhanchen/unsloth-blackwell-docker
|
||||
#
|
||||
# Requires: docker, pigz (or gzip), hf (or huggingface-cli) authenticated
|
||||
# with a WRITE-scoped token: `hf auth login`.
|
||||
set -euo pipefail
|
||||
|
||||
IMAGE="${1:?usage: hf_push.sh <image:tag> <hf_repo>}"
|
||||
REPO="${2:?usage: hf_push.sh <image:tag> <hf_repo>}"
|
||||
TAG="${IMAGE##*:}"
|
||||
NAME="${IMAGE%:*}"
|
||||
NAME="${NAME##*/}"
|
||||
BLOB="${NAME}-${TAG}.tar.gz"
|
||||
WORK="${HF_PUSH_TMP:-/tmp}"
|
||||
|
||||
command -v docker >/dev/null || { echo "ERROR: docker not on PATH"; exit 1; }
|
||||
|
||||
# Prefer the new `hf` CLI. The old `huggingface-cli` was deprecated in
|
||||
# huggingface_hub >= 0.27 and silently exits with a deprecation notice
|
||||
# instead of doing the upload.
|
||||
if command -v hf >/dev/null 2>&1; then
|
||||
HF_CMD=(hf upload)
|
||||
elif command -v huggingface-cli >/dev/null 2>&1; then
|
||||
echo "WARN: 'hf' not found, falling back to 'huggingface-cli' (deprecated)" >&2
|
||||
HF_CMD=(huggingface-cli upload)
|
||||
else
|
||||
echo "ERROR: need 'hf' (pip install -U huggingface_hub) or 'huggingface-cli'"; exit 1
|
||||
fi
|
||||
COMPRESSOR=$(command -v pigz || command -v gzip) || { echo "ERROR: need pigz or gzip"; exit 1; }
|
||||
|
||||
OUT="${WORK}/${BLOB}"
|
||||
echo ">> saving ${IMAGE} -> ${OUT} (using ${COMPRESSOR##*/})"
|
||||
docker save "${IMAGE}" | "${COMPRESSOR}" > "${OUT}"
|
||||
ls -lh "${OUT}"
|
||||
|
||||
echo ">> uploading to https://huggingface.co/${REPO}/blob/main/${BLOB} (via: ${HF_CMD[*]})"
|
||||
"${HF_CMD[@]}" "${REPO}" "${OUT}" "${BLOB}" \
|
||||
--repo-type=model \
|
||||
--commit-message="push ${IMAGE} ($(docker inspect --format '{{.Id}}' "${IMAGE}" | cut -c8-19))"
|
||||
|
||||
echo ">> pushed."
|
||||
echo "On the consumer side, run:"
|
||||
echo " bash docker/hf_pull.sh ${REPO} ${BLOB} ${IMAGE}"
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# One-time host setup: register QEMU binfmt handlers so `docker buildx` can
|
||||
# build images for foreign architectures (e.g. linux/arm64 on an x86_64 host).
|
||||
#
|
||||
# After this runs once per host reboot you can do:
|
||||
#
|
||||
# docker buildx build --platform linux/arm64 -t unsloth-blackwell:arm64 .
|
||||
# docker buildx build --platform linux/amd64,linux/arm64 --push -t YOU/img:tag .
|
||||
#
|
||||
# Important: QEMU is used at BUILD time only. The resulting arm64 image must
|
||||
# be RUN on an aarch64 host (e.g. DGX Spark / GB10) -- CUDA does not work under
|
||||
# runtime emulation. To smoke-test the arm64 image you need an actual arm64
|
||||
# GPU machine.
|
||||
#
|
||||
# Usage:
|
||||
# bash docker/setup_qemu.sh
|
||||
#
|
||||
# Requires: docker (28+ recommended), docker buildx plugin, root via sudo or
|
||||
# membership in the `docker` group. No network access to NVIDIA registries
|
||||
# is needed for this step.
|
||||
set -euo pipefail
|
||||
|
||||
command -v docker >/dev/null || { echo "ERROR: docker not on PATH"; exit 1; }
|
||||
docker buildx version >/dev/null 2>&1 || {
|
||||
echo "ERROR: 'docker buildx' missing. Install:" >&2
|
||||
echo " Ubuntu/Debian: sudo apt-get install -y docker-buildx" >&2
|
||||
echo " RHEL/Fedora: sudo dnf install -y docker-buildx-plugin" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
ARCH="$(uname -m)"
|
||||
echo ">> host arch: ${ARCH}"
|
||||
|
||||
# `tonistiigi/binfmt --install all` registers handlers for every supported
|
||||
# foreign arch; harmless if some are already registered. This is the canonical
|
||||
# upstream Docker recipe; see https://docs.docker.com/build/building/multi-platform/
|
||||
echo ">> registering QEMU binfmt handlers via tonistiigi/binfmt..."
|
||||
docker run --privileged --rm tonistiigi/binfmt --install all
|
||||
|
||||
# Ensure we have a buildx builder that can target multiple platforms.
|
||||
# The default 'docker' driver builder is single-platform; we create (or
|
||||
# reuse) a 'unsloth-multiarch' container-driver builder which is multi-arch.
|
||||
BUILDER="unsloth-multiarch"
|
||||
if docker buildx inspect "${BUILDER}" >/dev/null 2>&1; then
|
||||
echo ">> buildx builder '${BUILDER}' already exists"
|
||||
else
|
||||
echo ">> creating buildx builder '${BUILDER}'"
|
||||
docker buildx create --name "${BUILDER}" --driver docker-container --use
|
||||
fi
|
||||
docker buildx use "${BUILDER}"
|
||||
docker buildx inspect --bootstrap "${BUILDER}" | sed -n '1,12p'
|
||||
|
||||
echo
|
||||
echo ">> done. Verify with:"
|
||||
echo " docker buildx ls"
|
||||
echo " docker buildx inspect ${BUILDER}"
|
||||
echo
|
||||
echo ">> cross-arch build example:"
|
||||
echo " docker buildx build --platform linux/arm64 -t unsloth-blackwell:arm64 docker/"
|
||||
|
|
@ -1,410 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
# End-to-end Docker validation for the unsloth-blackwell image.
|
||||
#
|
||||
# Runs three blocks:
|
||||
# 1. Host pre-flight (docker, nvidia-smi, nvidia runtime registered)
|
||||
# 2. Build the image (no GPU required at build time)
|
||||
# 3a. Smoke test: 5-step LoRA on Llama-3.2-1B (~1-2 min)
|
||||
# 3b. Real workload: gpt-oss-20B fine-tuning notebook with max_steps=10
|
||||
# (~10 min, needs ~30GB free for the model cache)
|
||||
#
|
||||
# Usage:
|
||||
# bash docker/test_locally.sh # all blocks (native arch)
|
||||
# bash docker/test_locally.sh --skip-notebook # blocks 1-3a only (fast)
|
||||
# bash docker/test_locally.sh --skip-build # assume $TAG already built
|
||||
# bash docker/test_locally.sh --platform arm64 # cross-build for DGX Spark
|
||||
# # (auto-skips smoke/notebook)
|
||||
# TAG=my-image:latest bash docker/test_locally.sh
|
||||
# HF_TOKEN=hf_xxx bash docker/test_locally.sh # for gated models (optional)
|
||||
#
|
||||
# All output is teed to $LOG_DIR (default /tmp/unsloth-docker-test/).
|
||||
# Paste the listed log snippets back if anything fails.
|
||||
set -uo pipefail
|
||||
|
||||
TAG="${TAG:-unsloth-blackwell:test}"
|
||||
LOG_DIR="${LOG_DIR:-/tmp/unsloth-docker-test}"
|
||||
SKIP_BUILD=0
|
||||
SKIP_NOTEBOOK=0
|
||||
# Platform selector. Empty = let buildx default to the host arch (no
|
||||
# --platform passed). "amd64" / "arm64" = single-arch cross-build via QEMU
|
||||
# (requires `bash docker/setup_qemu.sh` to have been run once).
|
||||
PLATFORM=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--skip-build) SKIP_BUILD=1; shift ;;
|
||||
--skip-notebook) SKIP_NOTEBOOK=1; shift ;;
|
||||
--tag) TAG="$2"; shift 2 ;;
|
||||
--log-dir) LOG_DIR="$2"; shift 2 ;;
|
||||
--platform)
|
||||
case "$2" in
|
||||
amd64|arm64|linux/amd64|linux/arm64) PLATFORM="${2#linux/}" ;;
|
||||
*) echo "ERROR: --platform must be amd64 or arm64 (got '$2')" >&2; exit 2 ;;
|
||||
esac
|
||||
shift 2
|
||||
;;
|
||||
--help|-h) sed -n '2,22p' "$0"; exit 0 ;;
|
||||
*) echo "Unknown flag: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# When cross-building, the resulting image cannot be exercised on this host
|
||||
# (CUDA does not work under QEMU runtime emulation). Auto-skip the GPU blocks
|
||||
# and warn the user. They can paste back the build log either way to prove
|
||||
# the wheels resolve + the build-time torch._C._cuda_getArchFlags() assertion
|
||||
# passes on the foreign arch.
|
||||
HOST_ARCH="$(uname -m)"
|
||||
case "${HOST_ARCH}" in
|
||||
x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;;
|
||||
aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;;
|
||||
*) HOST_DOCKER_ARCH="${HOST_ARCH}" ;;
|
||||
esac
|
||||
CROSS_ARCH=0
|
||||
if [[ -n "${PLATFORM}" && "${PLATFORM}" != "${HOST_DOCKER_ARCH}" ]]; then
|
||||
CROSS_ARCH=1
|
||||
fi
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
GREEN='\033[1;32m'; RED='\033[1;31m'; YELLOW='\033[1;33m'; BLUE='\033[1;34m'; NC='\033[0m'
|
||||
banner() { printf "\n${BLUE}==== %s ====${NC}\n" "$*"; }
|
||||
ok() { printf "${GREEN}OK${NC} %s\n" "$*"; }
|
||||
warn() { printf "${YELLOW}WARN${NC} %s\n" "$*"; }
|
||||
err() { printf "${RED}ERROR${NC} %s\n" "$*" >&2; }
|
||||
fail() { err "$*"; exit 1; }
|
||||
|
||||
# ============================================================================
|
||||
# Block 1: pre-flight
|
||||
# ============================================================================
|
||||
banner "Block 1: host pre-flight"
|
||||
|
||||
command -v docker >/dev/null 2>&1 || fail "docker not found on PATH"
|
||||
echo " docker: $(docker --version)"
|
||||
|
||||
# Verify we can talk to the docker daemon as the current user -- catches the
|
||||
# "user not in docker group" case up front, instead of a later buildx
|
||||
# "permission denied on /var/run/docker.sock" that masquerades as a build failure.
|
||||
DOCKER_INFO_OUT=$(docker info 2>&1)
|
||||
DOCKER_INFO_RC=$?
|
||||
if [[ $DOCKER_INFO_RC -ne 0 ]]; then
|
||||
err "Cannot talk to the docker daemon as user '$USER'."
|
||||
cat >&2 <<MSG
|
||||
|
||||
docker info exited $DOCKER_INFO_RC. The most common cause is that your user
|
||||
is not in the 'docker' group. Fix:
|
||||
|
||||
sudo usermod -aG docker \$USER
|
||||
newgrp docker # activate the new group in this shell
|
||||
docker info | head -3 # verify
|
||||
|
||||
Then re-run this script in the same shell (or any new login session).
|
||||
|
||||
Alternative: run the script with sudo, but be aware it will use root's
|
||||
home directory for HF cache (~/root/.cache/huggingface) which is probably
|
||||
not what you want.
|
||||
|
||||
Raw docker info output:
|
||||
$DOCKER_INFO_OUT
|
||||
MSG
|
||||
fail "docker daemon unreachable"
|
||||
fi
|
||||
echo " daemon: reachable as '$USER'"
|
||||
|
||||
if command -v nvidia-smi >/dev/null 2>&1; then
|
||||
echo " host gpu: $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)"
|
||||
echo " host driver: $(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -1)"
|
||||
else
|
||||
warn "nvidia-smi not on the host -- you may not be able to run --gpus all"
|
||||
fi
|
||||
|
||||
# This grep only makes sense once we know `docker info` succeeded above.
|
||||
if echo "$DOCKER_INFO_OUT" | grep -qiE 'Runtimes:.*nvidia'; then
|
||||
echo " nvidia runtime: registered with docker"
|
||||
else
|
||||
warn "docker info does not list 'nvidia' as a runtime"
|
||||
warn "(on Docker 28+ with CDI this is often a false positive; the real"
|
||||
warn " test is whether --gpus all works in Block 3a below)"
|
||||
warn "if --gpus all fails, install nvidia-container-toolkit:"
|
||||
warn " https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html"
|
||||
warn " then: sudo systemctl restart docker"
|
||||
fi
|
||||
ok "pre-flight done"
|
||||
|
||||
# ============================================================================
|
||||
# Block 2: build
|
||||
# ============================================================================
|
||||
if [[ $SKIP_BUILD -eq 1 ]]; then
|
||||
warn "skipping build (--skip-build); expecting $TAG to exist"
|
||||
else
|
||||
banner "Block 2: build $TAG"
|
||||
|
||||
# Find the build context: current dir, docker/ subdir, or clone the PR branch
|
||||
if [[ -f "Dockerfile" && -f "smoke_test.py" ]]; then
|
||||
BUILD_CTX="$PWD"
|
||||
elif [[ -f "docker/Dockerfile" ]]; then
|
||||
BUILD_CTX="$PWD/docker"
|
||||
else
|
||||
BUILD_CTX="/tmp/unsloth-pr/docker"
|
||||
if [[ ! -d /tmp/unsloth-pr/.git ]]; then
|
||||
echo " cloning docker-blackwell-build branch..."
|
||||
if ! git clone --depth 1 -b docker-blackwell-build \
|
||||
https://github.com/unslothai/unsloth.git /tmp/unsloth-pr 2>&1 | tail -3; then
|
||||
fail "could not clone docker-blackwell-build into /tmp/unsloth-pr; refusing to build from stale context"
|
||||
fi
|
||||
else
|
||||
# `set -e` is not active in this script, so a failing pull would
|
||||
# otherwise be silently masked and we'd build from a stale clone.
|
||||
# Explicitly fail loudly when the fast-forward refresh cannot run.
|
||||
if ! git -C /tmp/unsloth-pr pull --ff-only 2>&1 | tail -2; then
|
||||
fail "git pull --ff-only failed in /tmp/unsloth-pr; refusing to build from stale context (delete /tmp/unsloth-pr to reclone)"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
echo " build context: $BUILD_CTX"
|
||||
|
||||
BUILD_LOG="$LOG_DIR/build.log"
|
||||
echo " log: $BUILD_LOG"
|
||||
|
||||
# The Dockerfile uses BuildKit-only features ('# syntax=docker/dockerfile:1.7'
|
||||
# and 'RUN ... <<\'PY\'' heredocs). Docker 28 removed the legacy builder
|
||||
# entirely -- DOCKER_BUILDKIT=1 now delegates to buildx, so without the
|
||||
# buildx component installed there is no fallback that works. Fail fast
|
||||
# with install instructions before attempting the build.
|
||||
if ! docker buildx version >/dev/null 2>&1; then
|
||||
cat >&2 <<'MSG'
|
||||
|
||||
ERROR: docker buildx is not installed.
|
||||
|
||||
The Dockerfile requires BuildKit (syntax=docker/dockerfile:1.7 + RUN heredocs).
|
||||
Docker 28 removed the legacy builder, so buildx is required for any build.
|
||||
|
||||
Install buildx, then re-run this script:
|
||||
|
||||
Ubuntu / Debian (apt):
|
||||
sudo apt-get update && sudo apt-get install -y docker-buildx
|
||||
|
||||
Ubuntu / Debian (Docker's official repo, recommended):
|
||||
# Follow https://docs.docker.com/engine/install/ubuntu/ -- the docker-ce
|
||||
# package bundles docker-buildx-plugin and is what most production guides
|
||||
# assume. The Ubuntu-shipped docker.io package omits buildx.
|
||||
|
||||
RHEL / Fedora (dnf):
|
||||
sudo dnf install -y docker-buildx-plugin
|
||||
|
||||
Manual install (any distro):
|
||||
https://github.com/docker/buildx/releases (download into ~/.docker/cli-plugins/)
|
||||
|
||||
Verify with: docker buildx version
|
||||
MSG
|
||||
fail "docker buildx required -- install per the message above"
|
||||
fi
|
||||
echo " builder: docker buildx ($(docker buildx version | head -1))"
|
||||
|
||||
BUILD_ARGS=( --progress=plain )
|
||||
if [[ -n "${PLATFORM}" ]]; then
|
||||
echo " platform: linux/${PLATFORM}"
|
||||
BUILD_ARGS+=( --platform "linux/${PLATFORM}" )
|
||||
if [[ ${CROSS_ARCH} -eq 1 ]]; then
|
||||
echo " cross-build: yes (host=${HOST_DOCKER_ARCH}); verifying QEMU binfmt..."
|
||||
if ! docker run --rm --privileged tonistiigi/binfmt 2>/dev/null \
|
||||
| grep -q "\"linux/${PLATFORM}\""; then
|
||||
cat >&2 <<MSG
|
||||
|
||||
ERROR: QEMU binfmt handler for linux/${PLATFORM} is not registered.
|
||||
|
||||
Run the one-time host setup first:
|
||||
|
||||
bash docker/setup_qemu.sh
|
||||
|
||||
Then re-run this script.
|
||||
MSG
|
||||
fail "QEMU binfmt missing for linux/${PLATFORM}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " platform: (native, no --platform)"
|
||||
fi
|
||||
# --load works only for single-platform builds; we never multi-platform here.
|
||||
BUILD_ARGS+=( --load )
|
||||
|
||||
docker buildx build "${BUILD_ARGS[@]}" -t "$TAG" "$BUILD_CTX" 2>&1 | tee "$BUILD_LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
fail "docker build exited $rc -- see $BUILD_LOG"
|
||||
fi
|
||||
|
||||
# Sanity check the build's own self-test ran and passed
|
||||
if grep -q "FAIL: missing wheels\|sm_100 (B200/GB200) missing\|sm_120 (RTX 5090) missing on amd64\|no Blackwell consumer SASS" "$BUILD_LOG"; then
|
||||
fail "build-time sanity check failed -- see $BUILD_LOG"
|
||||
fi
|
||||
grep -E "OK: torch 2.11.0|OK: all required wheels|import cleanly on no-GPU host" "$BUILD_LOG" || \
|
||||
warn "could not find 'OK:' lines in build log -- did the verification step run?"
|
||||
ok "built $TAG"
|
||||
fi
|
||||
|
||||
# When the image we just built (or were told to use) does not match the host
|
||||
# architecture, the smoke test and notebook blocks would attempt to launch
|
||||
# foreign-arch user-space under QEMU plus --gpus all -- which is broken by
|
||||
# design: nvidia-container-toolkit cannot expose a GPU to a QEMU-emulated
|
||||
# guest, and even if it could, CUDA kernels do not run under user-space CPU
|
||||
# emulation. Skip those blocks with a loud warning so the user doesn't think
|
||||
# they're seeing a real validation pass.
|
||||
if [[ ${CROSS_ARCH} -eq 1 ]]; then
|
||||
warn "cross-arch build (host=${HOST_DOCKER_ARCH}, image=${PLATFORM})."
|
||||
warn "skipping smoke test + notebook -- CUDA does not work under QEMU runtime."
|
||||
warn "to validate end-to-end on linux/${PLATFORM}, transfer the image to an"
|
||||
warn "actual ${PLATFORM} host (e.g. DGX Spark for arm64) and re-run with --skip-build."
|
||||
banner "summary"
|
||||
echo " image: $TAG"
|
||||
echo " platform: linux/${PLATFORM} (cross-built on ${HOST_DOCKER_ARCH})"
|
||||
echo " log dir: $LOG_DIR"
|
||||
echo
|
||||
[[ $SKIP_BUILD -eq 0 ]] && echo " to paste back for PR validation:"
|
||||
[[ $SKIP_BUILD -eq 0 ]] && echo " tail -80 $LOG_DIR/build.log"
|
||||
ok "cross-arch build verified (wheels + arch-flags assertion passed)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# Block 3a: smoke test
|
||||
# ============================================================================
|
||||
banner "Block 3a: smoke test (5-step LoRA on Llama-3.2-1B)"
|
||||
SMOKE_LOG="$LOG_DIR/smoke.log"
|
||||
echo " log: $SMOKE_LOG"
|
||||
docker run --rm --gpus all "$TAG" python /workspace/smoke_test.py 2>&1 | tee "$SMOKE_LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
fail "smoke test exited $rc -- see $SMOKE_LOG"
|
||||
fi
|
||||
if ! grep -q "all checks passed" "$SMOKE_LOG"; then
|
||||
fail "smoke test did not print 'all checks passed' -- see $SMOKE_LOG"
|
||||
fi
|
||||
ok "smoke test passed"
|
||||
|
||||
# ============================================================================
|
||||
# Block 3b: gpt-oss-20B fine-tuning notebook
|
||||
# ============================================================================
|
||||
if [[ $SKIP_NOTEBOOK -eq 1 ]]; then
|
||||
warn "skipping gpt-oss-20B notebook (--skip-notebook)"
|
||||
else
|
||||
banner "Block 3b: gpt-oss-20B fine-tuning notebook (10 LoRA steps)"
|
||||
GPT_LOG="$LOG_DIR/gpt_oss.log"
|
||||
HOST_RUN_DIR="$LOG_DIR/host"
|
||||
mkdir -p "$HOST_RUN_DIR"
|
||||
echo " log: $GPT_LOG"
|
||||
echo " host dir: $HOST_RUN_DIR"
|
||||
|
||||
cat > "$HOST_RUN_DIR/run_notebook.sh" <<'INNER'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
cd /workspace/host
|
||||
|
||||
echo "=== install triton_kernels (MXFP4 support for unsloth/gpt-oss-20b) ==="
|
||||
pip install -q 'git+https://github.com/triton-lang/triton.git@0add68262ab0a2e33b84524346cb27cbb2787356#subdirectory=python/triton_kernels' 2>&1 | tail -5
|
||||
|
||||
echo
|
||||
echo "=== fetch + convert notebook ==="
|
||||
# Use nbformat directly. We then post-process to:
|
||||
# 1. Skip install cells -- the container already has unsloth + deps baked in;
|
||||
# the notebook's install cell uses Jupyter !shell magic (raw `!pip install
|
||||
# ...` lines) that nbformat dumps verbatim and Python cannot parse.
|
||||
# 2. Comment out any stray !cmd / %magic lines in non-install cells.
|
||||
pip install -q nbformat
|
||||
# Pin to an immutable commit so this validation script doesn't silently
|
||||
# change semantics when notebooks/main rolls forward. Bump deliberately
|
||||
# when the upstream notebook gets a fix you want to verify against.
|
||||
NB_REPO_REF="${NB_REPO_REF:-efe20c97a5bba3088b25fe068a4b1c98c0cf3a3a}"
|
||||
curl -fsSL "https://raw.githubusercontent.com/unslothai/notebooks/${NB_REPO_REF}/nb/gpt-oss-(20B)-Fine-tuning.ipynb" -o nb.ipynb
|
||||
test -s nb.ipynb || { echo "FAIL: nb.ipynb was not downloaded"; exit 1; }
|
||||
python - <<'PY'
|
||||
import nbformat, re
|
||||
nb = nbformat.read('nb.ipynb', as_version=4)
|
||||
out, skipped = [], 0
|
||||
INSTALL_MARKERS = (
|
||||
"pip install", "uv pip install", "apt-get install",
|
||||
"_original_packages", "COLAB_", "importlib.util.find_spec",
|
||||
)
|
||||
for c in nb.cells:
|
||||
if c.cell_type != "code":
|
||||
continue
|
||||
src = c.source or ""
|
||||
if any(m in src for m in INSTALL_MARKERS):
|
||||
skipped += 1
|
||||
first = next((ln for ln in src.splitlines() if ln.strip()), "")[:80]
|
||||
out.append(f"# (skipped install/setup cell: {first!r})")
|
||||
out.append("")
|
||||
continue
|
||||
for line in src.splitlines():
|
||||
stripped = line.lstrip()
|
||||
if stripped.startswith(("!", "%")):
|
||||
out.append(f"# (jupyter magic stripped) {line}")
|
||||
else:
|
||||
out.append(line)
|
||||
out.append("")
|
||||
with open("nb.py", "w") as f:
|
||||
f.write("\n".join(out) + "\n")
|
||||
print(f" converted nb.py: {sum(1 for _ in open('nb.py'))} lines, {skipped} install cell(s) skipped")
|
||||
PY
|
||||
test -s nb.py || { echo "FAIL: nb.py was not produced"; exit 1; }
|
||||
# Sanity-check: nb.py must parse as valid Python before we try to run it.
|
||||
python -c "import ast; ast.parse(open('nb.py').read()); print(' nb.py is valid Python')"
|
||||
|
||||
echo
|
||||
echo "=== patch nb.py: max_steps 30 -> 10, drop pre-train demo generations ==="
|
||||
python - <<'PY'
|
||||
import re
|
||||
src = open('nb.py').read()
|
||||
src = src.replace('max_steps = 30', 'max_steps = 10')
|
||||
src = re.sub(
|
||||
r'messages = \[\s*\{[\"\']role[\"\']: [\"\']user[\"\'], [\"\']content[\"\']: [\"\']Solve x\^5.*?\n_ = model\.generate.*?streamer = TextStreamer\(tokenizer\)\)\n',
|
||||
'# (pre-train inference skipped)\n',
|
||||
src, flags=re.DOTALL, count=3,
|
||||
)
|
||||
open('nb.py', 'w').write(src)
|
||||
print(' patched. max_steps now:', re.search(r'max_steps = (\d+)', src).group(1))
|
||||
PY
|
||||
|
||||
echo
|
||||
echo "=== run gpt-oss-20B fine-tuning ==="
|
||||
python -u nb.py
|
||||
INNER
|
||||
chmod +x "$HOST_RUN_DIR/run_notebook.sh"
|
||||
|
||||
# Only forward HF_TOKEN if the host has one set, so an empty
|
||||
# `-e HF_TOKEN=` does not shadow whatever is already inside the image.
|
||||
# Use the dash-only form `-e HF_TOKEN` so the secret value never
|
||||
# lands in argv (visible via /proc/<pid>/cmdline to any user on
|
||||
# the host for the lifetime of the docker CLI process).
|
||||
HF_ARGS=()
|
||||
[[ -n "${HF_TOKEN:-}" ]] && HF_ARGS+=(-e HF_TOKEN)
|
||||
docker run --rm \
|
||||
--gpus all \
|
||||
--ipc=host \
|
||||
--ulimit memlock=-1 \
|
||||
--ulimit stack=67108864 \
|
||||
-v "$HOST_RUN_DIR:/workspace/host" \
|
||||
-v "$HOME/.cache/huggingface:/workspace/.cache/huggingface" \
|
||||
${HF_ARGS[@]+"${HF_ARGS[@]}"} \
|
||||
-e HF_HUB_ENABLE_HF_TRANSFER=1 \
|
||||
"$TAG" \
|
||||
bash /workspace/host/run_notebook.sh 2>&1 | tee "$GPT_LOG"
|
||||
rc=${PIPESTATUS[0]}
|
||||
if [[ $rc -ne 0 ]]; then
|
||||
fail "gpt-oss-20B notebook exited $rc -- see $GPT_LOG"
|
||||
fi
|
||||
ok "gpt-oss-20B notebook completed"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# Summary
|
||||
# ============================================================================
|
||||
banner "summary"
|
||||
echo " image: $TAG"
|
||||
echo " log dir: $LOG_DIR"
|
||||
echo
|
||||
echo " to paste back for PR validation:"
|
||||
[[ $SKIP_BUILD -eq 0 ]] && echo " tail -40 $LOG_DIR/build.log"
|
||||
echo " cat $LOG_DIR/smoke.log"
|
||||
[[ $SKIP_NOTEBOOK -eq 0 ]] && echo " tail -100 $LOG_DIR/gpt_oss.log"
|
||||
echo
|
||||
ok "all blocks completed"
|
||||
Loading…
Add table
Add a link
Reference in a new issue