diff --git a/docker/docker_confirm.ps1 b/docker/docker_confirm.ps1 deleted file mode 100644 index f5388772e6..0000000000 --- a/docker/docker_confirm.ps1 +++ /dev/null @@ -1,233 +0,0 @@ -# docker_confirm.ps1 (Unsloth Docker image confirmation - Windows) -# Confirms the published Unsloth Docker images actually work on this machine -# through Docker Desktop: pulls them, checks WSL2 GPU passthrough (or CPU -# fallback), runs a real 5-step LoRA training smoke, checks the baked -# llama.cpp GGUF tooling, boots the full image and probes Studio + -# JupyterLab, then prints a PASS/FAIL report. -# -# One-liner (PowerShell): -# irm https://raw.githubusercontent.com/unslothai/unsloth/main/docker/docker_confirm.ps1 | iex -# -# What to expect per machine class: -# Windows + NVIDIA (RTX 5070 / DGX Spark): GPU mode when Docker Desktop -# uses the WSL2 backend with GPU support enabled (Settings > Resources). -# Windows + AMD (Strix Halo): CPU mode - Docker Desktop has no ROCm -# passthrough; training phases are skipped, Studio chat / Jupyter / GGUF -# tooling still validate. Use the native install for AMD GPU work. -# -# Env overrides: $env:IMAGE, $env:BASE_IMAGE, $env:GPUS ('auto'|'all'|'none'), -# $env:PORT_STUDIO (18000), $env:PORT_JUPYTER (18888), $env:WORK, -# $env:SKIP_PULL, $env:SKIP_TRAIN, $env:KEEP - -$ErrorActionPreference = "Continue" -$IMAGE = if ($env:IMAGE) { $env:IMAGE } else { "unsloth/unsloth:latest" } -$BASE_IMAGE = if ($env:BASE_IMAGE) { $env:BASE_IMAGE } else { "unsloth/unsloth:core" } -$GPUS = if ($env:GPUS) { $env:GPUS } else { "auto" } -$PORT_STUDIO = if ($env:PORT_STUDIO) { $env:PORT_STUDIO } else { 18000 } -$PORT_JUPYTER = if ($env:PORT_JUPYTER) { $env:PORT_JUPYTER } else { 18888 } -$WORK = if ($env:WORK) { $env:WORK } else { Join-Path $HOME "unsloth_docker_test" } -$SKIP_PULL = $env:SKIP_PULL -eq "1" -$SKIP_TRAIN = $env:SKIP_TRAIN -eq "1" -$KEEP = $env:KEEP -eq "1" - -$script:PASS_N = 0; $script:FAIL_N = 0; $script:WARN_N = 0; $script:STUDIO_CID = "" -function Bold($m){ Write-Host $m -ForegroundColor White } -function Ok($m) { Write-Host " [PASS] $m" -ForegroundColor Green; $script:PASS_N++ } -function Bad($m) { Write-Host " [FAIL] $m" -ForegroundColor Red; $script:FAIL_N++ } -function Warn($m){ Write-Host " [WARN] $m" -ForegroundColor Yellow; $script:WARN_N++ } -function Info($m){ Write-Host " $m" } -function Hr() { Write-Host ("-" * 63) } - -New-Item -ItemType Directory -Force -Path $WORK | Out-Null -Write-Host ""; Bold "=== Unsloth Docker image confirmation (Windows) ===" -Write-Host "scratch dir : $WORK"; Hr - -# 1) Host detection ----------------------------------------------------------- -Bold "1) Host detection" -Info ("windows : " + [System.Environment]::OSVersion.VersionString + " " + $env:PROCESSOR_ARCHITECTURE) -if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { - Bad "docker not found - install Docker Desktop first" - Bold "RESULT: cannot continue without docker."; exit 1 -} -docker info *> $null -if ($LASTEXITCODE -ne 0) { - Bad "docker daemon not reachable - start Docker Desktop" - Bold "RESULT: cannot continue without a reachable docker daemon."; exit 1 -} -Ok ("docker daemon reachable (" + (docker --version) + ")") -$osType = (docker info --format "{{.OSType}}" 2>$null) -if ($osType -ne "linux") { - Bad "Docker Desktop is in Windows-container mode (OSType=$osType) - switch to Linux containers" -} - -$GPU_MODE = $false -if ($GPUS -eq "none") { - Info "GPU mode : disabled by GPUS=none" -} elseif (Get-Command nvidia-smi -ErrorAction SilentlyContinue) { - $gpus = nvidia-smi --query-gpu=index,name,compute_cap --format=csv,noheader 2>$null - if ($LASTEXITCODE -eq 0 -and $gpus) { - $gpus | ForEach-Object { Info (" - " + $_) } - Ok "NVIDIA GPU visible on the host - probing WSL2 passthrough below" - $GPU_MODE = $true - } else { - Info "nvidia-smi present but no GPU listed" - } -} else { - Info "no NVIDIA GPU on the host (or nvidia-smi missing)" -} -if (-not $GPU_MODE) { - Warn "CPU mode: training phases are skipped; Studio chat / Jupyter / GGUF tooling still validate" -} -Hr - -# 2) Pull images -------------------------------------------------------------- -Bold "2) Pull images" -foreach ($img in @($BASE_IMAGE, $IMAGE)) { - if ($SKIP_PULL) { - docker image inspect $img *> $null - if ($LASTEXITCODE -eq 0) { Ok "local image present: $img" } else { Bad "SKIP_PULL=1 but image missing locally: $img" } - } else { - $log = Join-Path $WORK ("pull_" + ($img -replace "[/:]", "_") + ".log") - docker pull $img *> $log - if ($LASTEXITCODE -eq 0) { Ok "pulled $img" } - else { - docker image inspect $img *> $null - # Locally built tags are not on a registry; presence is what matters. - if ($LASTEXITCODE -eq 0) { Warn "not pullable but present locally: $img" } - else { Bad "could not pull $img (see $log)" } - } - } -} -Hr - -# 3) Container runtime check -------------------------------------------------- -Bold "3) Container runtime check" -# Mirror docker_confirm.sh's GPU selector translation: bare indices and -# comma lists become device= selectors (Docker reads a bare integer for -# --gpus as a COUNT, not an index). Built as an args array so every docker -# run call splats it identically. -# -# Comma lists are special: docker CSV-parses the --gpus value, so a list -# must arrive as a literal "device=0,1" INCLUDING the double quotes. How -# PowerShell passes embedded quotes to native commands changed in 7.3 -# (PSNativeCommandArgumentPassing), so pick the escaping per version; -# single selectors need no quoting anywhere. -$GPU_SELECTOR = "all" -if ($GPUS -notin @("auto", "all", "none")) { - $sel = $GPUS -replace "^device=", "" - if ($sel -match ",") { - if ($PSVersionTable.PSVersion -ge [version]"7.3") { $GPU_SELECTOR = '"device=' + $sel + '"' } - else { $GPU_SELECTOR = '\"device=' + $sel + '\"' } - } else { - $GPU_SELECTOR = "device=$sel" - } -} -$GpuRunArgs = @("--gpus", $GPU_SELECTOR) -if ($GPU_MODE) { - $log = Join-Path $WORK "gpu_check.log" - docker run --rm @GpuRunArgs $BASE_IMAGE python -c "import torch; assert torch.cuda.is_available(); print('torch', torch.__version__, '-', torch.cuda.get_device_name(0))" *> $log - if ($LASTEXITCODE -eq 0) { - Ok ("torch.cuda available in-container: " + (Get-Content $log -Tail 1)) - } else { - Bad "GPU passthrough failed (see $log) - check Docker Desktop WSL2 GPU support; falling back to CPU mode" - Get-Content $log -Tail 5 | ForEach-Object { Info $_ } - $GPU_MODE = $false - } -} -if (-not $GPU_MODE) { - $log = Join-Path $WORK "cpu_check.log" - docker run --rm -e UNSLOTH_ALLOW_CPU=1 $BASE_IMAGE python -c "import torch; print('torch', torch.__version__, 'cpu-mode ok')" *> $log - if ($LASTEXITCODE -eq 0) { - Ok ("CPU mode boots: " + (Get-Content $log -Tail 1)) - } else { - Bad "container failed to start even in CPU mode (see $log)" - Get-Content $log -Tail 5 | ForEach-Object { Info $_ } - } -} -Hr - -# 4) Training smoke (GPU only) ------------------------------------------------ -Bold "4) Training smoke" -if ($GPU_MODE -and -not $SKIP_TRAIN) { - $log = Join-Path $WORK "train_smoke.log" - $hfArgs = @(); if ($env:HF_TOKEN) { $hfArgs = @("-e", "HF_TOKEN") } - docker run --rm @GpuRunArgs --ipc=host @hfArgs $BASE_IMAGE python /workspace/smoke_test.py *> $log - if ($LASTEXITCODE -eq 0) { - Ok "smoke_test.py: 5 LoRA steps completed" - Select-String -Path $log -Pattern "^step|loss" | Select-Object -Last 5 | ForEach-Object { Info $_.Line } - } else { - Bad "training smoke failed (see $log)" - Get-Content $log -Tail 10 | ForEach-Object { Info $_ } - } -} else { - Warn "skipped (CPU mode or SKIP_TRAIN=1)" -} -Hr - -# 5) GGUF tooling ------------------------------------------------------------- -Bold "5) GGUF tooling (baked llama.cpp)" -$log = Join-Path $WORK "gguf_check.log" -docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 $BASE_IMAGE bash -c 'set -e; test -x "$UNSLOTH_LLAMA_CPP_PATH/llama-quantize"; test -f "$UNSLOTH_LLAMA_CPP_PATH/convert_hf_to_gguf.py"; "$UNSLOTH_LLAMA_CPP_PATH/llama-server" --version 2>&1 | head -2' *> $log -if ($LASTEXITCODE -eq 0) { - Ok "llama-quantize + llama-server + convert_hf_to_gguf.py present and runnable" - Select-String -Path $log -Pattern "version" | Select-Object -First 2 | ForEach-Object { Info $_.Line } -} else { - Bad "baked llama.cpp check failed (see $log)" - Get-Content $log -Tail 5 | ForEach-Object { Info $_ } -} -Hr - -# 5b) vLLM (GRPO fast_inference=True) ----------------------------------------- -Bold "5b) vLLM (GRPO fast_inference=True)" -$log = Join-Path $WORK "vllm_check.log" -docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 $BASE_IMAGE python -c 'import vllm; print("vllm", vllm.__version__)' *> $log -if ($LASTEXITCODE -eq 0) { - Ok ("vllm importable: " + (Get-Content $log -Tail 1)) -} else { - $imgArch = docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 $BASE_IMAGE uname -m 2>$null - if ($imgArch -eq "x86_64") { - Bad "vllm missing or broken on x86_64 image (see $log)" - Get-Content $log -Tail 3 | ForEach-Object { Info $_ } - } else { - Warn "vllm not available on $imgArch image; GRPO fast_inference=True unavailable (arm64 wheels are newer, fail-soft at image build)" - } -} -Hr - -# 6) Studio + JupyterLab ------------------------------------------------------ -Bold "6) Studio + JupyterLab (full image)" -$runArgs = @("-d", "-p", "${PORT_STUDIO}:8000", "-p", "${PORT_JUPYTER}:8888") -if ($GPU_MODE) { $runArgs += $GpuRunArgs } else { $runArgs += @("-e", "UNSLOTH_ALLOW_CPU=1") } -$script:STUDIO_CID = (docker run @runArgs $IMAGE 2>(Join-Path $WORK "studio_run.err")) -if (-not $script:STUDIO_CID) { - Bad ("full image failed to start (see " + (Join-Path $WORK "studio_run.err") + ")") -} else { - Info ("container : " + $script:STUDIO_CID.Substring(0, 12) + " (studio http://localhost:$PORT_STUDIO, jupyter http://localhost:$PORT_JUPYTER)") - $okStudio = $false; $okJupyter = $false - foreach ($i in 1..60) { - if (-not $okStudio) { try { Invoke-WebRequest -UseBasicParsing -Uri "http://localhost:$PORT_STUDIO/api/health" -TimeoutSec 4 | Out-Null; $okStudio = $true } catch {} } - # /login, not /api: a password hash is always configured so /api returns 403. - if (-not $okJupyter) { try { Invoke-WebRequest -UseBasicParsing -Uri "http://localhost:$PORT_JUPYTER/login" -TimeoutSec 4 | Out-Null; $okJupyter = $true } catch {} } - if ($okStudio -and $okJupyter) { break } - Start-Sleep -Seconds 5 - } - if ($okStudio) { Ok "Studio /api/health healthy" } else { Bad "Studio /api/health never went healthy (docker logs $($script:STUDIO_CID.Substring(0,12)))"; docker logs --tail 15 $script:STUDIO_CID 2>&1 | ForEach-Object { Info $_ } } - if ($okJupyter) { Ok "JupyterLab /login responding" } else { Bad "JupyterLab /login never responded" } -} -Hr - -# Summary --------------------------------------------------------------------- -Bold "=== SUMMARY ===" -Write-Host "images : $IMAGE / $BASE_IMAGE" -Write-Host ("gpu_mode : " + $GPU_MODE) -Write-Host "logs : $WORK" -Write-Host "PASS: $script:PASS_N WARN: $script:WARN_N FAIL: $script:FAIL_N" -if (-not $KEEP -and $script:STUDIO_CID) { docker rm -f $script:STUDIO_CID *> $null } -elseif ($KEEP -and $script:STUDIO_CID) { Write-Host ("container " + $script:STUDIO_CID.Substring(0,12) + " left running (KEEP=1): studio :$PORT_STUDIO jupyter :$PORT_JUPYTER") } -if ($script:FAIL_N -eq 0) { - Bold "RESULT: CONFIRMED - the Unsloth Docker images work on this machine." - exit 0 -} else { - Bold "RESULT: $script:FAIL_N hard failure(s) - paste this whole output back." - exit 1 -} diff --git a/docker/docker_confirm.sh b/docker/docker_confirm.sh deleted file mode 100644 index cef51691fc..0000000000 --- a/docker/docker_confirm.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env bash -# -# docker_confirm.sh (Unsloth Docker image confirmation - Linux / WSL2 / macOS) -# Confirms the published Unsloth Docker images actually work on this machine: -# pulls them, checks GPU passthrough (or CPU fallback), runs a real 5-step -# LoRA training smoke, checks the baked llama.cpp GGUF tooling, boots the -# full image and probes Studio + JupyterLab, then prints a PASS/FAIL report. -# -# Nothing is installed on the host beyond the Docker images themselves; the -# containers it starts are removed afterwards (KEEP=1 keeps them running). -# -# One-liner: -# curl -fsSL https://raw.githubusercontent.com/unslothai/unsloth/main/docker/docker_confirm.sh | bash -# -# What to expect per machine class: -# Linux + NVIDIA (B200 / RTX 6000 / RTX 50-series). GPU mode, all phases. -# Windows + NVIDIA via Docker Desktop (WSL2 backend): run inside the WSL2 -# distro or Git Bash. GPU mode if Docker Desktop has WSL2 GPU enabled. -# DGX Spark / GB10 (Linux arm64): GPU mode, the arm64 image child is pulled -# automatically. -# macOS (M-series) and Windows + AMD (Strix Halo): CPU mode is auto-detected -# (no NVIDIA passthrough exists for these); training phases are skipped, -# Studio chat / Jupyter / GGUF tooling still validate. -# -# Env overrides: IMAGE (default unsloth/unsloth:latest) -# BASE_IMAGE (default unsloth/unsloth:core) -# GPUS=all|none|0|0,1 (default: auto-detect) -# PORT_STUDIO=18000 PORT_JUPYTER=18888 -# WORK=~/unsloth_docker_test (logs) -# HF_CACHE=~/.cache/huggingface (mounted to speed model pulls) -# SKIP_PULL=1 (use local images) SKIP_TRAIN=1 KEEP=1 -# -set -uo pipefail - -IMAGE="${IMAGE:-unsloth/unsloth:latest}" -BASE_IMAGE="${BASE_IMAGE:-unsloth/unsloth:core}" -GPUS="${GPUS:-auto}" -PORT_STUDIO="${PORT_STUDIO:-18000}" -PORT_JUPYTER="${PORT_JUPYTER:-18888}" -WORK="${WORK:-$HOME/unsloth_docker_test}" -HF_CACHE="${HF_CACHE:-$HOME/.cache/huggingface}" -SKIP_PULL="${SKIP_PULL:-0}" -SKIP_TRAIN="${SKIP_TRAIN:-0}" -KEEP="${KEEP:-0}" -ARCH="$(uname -m)" -OS="$(uname -s)" - -PASS_N=0; FAIL_N=0; WARN_N=0; STUDIO_CID="" -bold(){ printf '\033[1m%s\033[0m\n' "$*"; } -ok(){ printf ' [PASS] %s\n' "$*"; PASS_N=$((PASS_N+1)); } -bad(){ printf ' [FAIL] %s\n' "$*"; FAIL_N=$((FAIL_N+1)); } -warn(){ printf ' [WARN] %s\n' "$*"; WARN_N=$((WARN_N+1)); } -info(){ printf ' %s\n' "$*"; } -hr(){ printf -- '---------------------------------------------------------------\n'; } - -cleanup(){ - if [ "$KEEP" != "1" ] && [ -n "$STUDIO_CID" ]; then - docker rm -f "$STUDIO_CID" >/dev/null 2>&1 - fi -} -trap cleanup EXIT - -mkdir -p "$WORK" "$HF_CACHE" -echo; bold "=== Unsloth Docker image confirmation ===" -echo "scratch dir : $WORK"; hr - -# --------------------------------------------------------------------------- # -# 1. Host detection -# --------------------------------------------------------------------------- # -bold "1) Host detection" -info "uname : $OS $ARCH ($(uname -r 2>/dev/null))" -IS_WSL=0 -grep -qiE "microsoft|wsl" /proc/version 2>/dev/null && { IS_WSL=1; info "WSL : yes"; } -if ! command -v docker >/dev/null 2>&1; then - bad "docker not found on PATH - install Docker Engine / Docker Desktop first" - echo; bold "RESULT: cannot continue without docker."; exit 1 -fi -if ! docker info >/dev/null 2>&1; then - bad "docker daemon not reachable (permission denied or not running)" - info "try: sudo usermod -aG docker \$USER && re-login, or start Docker Desktop" - echo; bold "RESULT: cannot continue without a reachable docker daemon."; exit 1 -fi -ok "docker daemon reachable ($(docker --version 2>/dev/null))" - -GPU_MODE=0 -NVRT_LISTED=0 -if [ "$GPUS" = "none" ]; then - info "GPU mode : disabled by GPUS=none" -elif command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q '^GPU'; then - info "GPU(s) :" - nvidia-smi --query-gpu=index,name,compute_cap --format=csv,noheader 2>/dev/null | sed 's/^/ - /' - # `docker info | grep Runtimes:.*nvidia` misses CDI setups (docker 25+ - # with nvidia-ctk cdi) and Docker Desktop's WSL2 backend, both of which - # expose GPUs without a host-visible runtime entry. Treat the listing as - # a hint only; phase 3 probes --gpus for real and demotes to CPU mode if - # the probe fails. - if docker info 2>/dev/null | grep -qi 'Runtimes:.*nvidia'; then - ok "NVIDIA GPU visible and docker lists the nvidia runtime" - NVRT_LISTED=1 - else - warn "nvidia runtime not listed by docker info (normal under CDI or Docker Desktop WSL2) - probing --gpus directly in phase 3" - fi - GPU_MODE=1 -else - info "no NVIDIA GPU on the host (or nvidia-smi missing)" -fi -if [ "$GPU_MODE" = "0" ]; then - warn "CPU mode: training phases are skipped; Studio chat / Jupyter / GGUF tooling still validate" -fi -GPU_FLAG=(--gpus all) -case "$GPUS" in - auto|all|none) ;; - *) GPU_FLAG=(--gpus "\"device=${GPUS}\"") ;; -esac -hr - -# --------------------------------------------------------------------------- # -# 2. Pull images -# --------------------------------------------------------------------------- # -bold "2) Pull images" -for img in "$BASE_IMAGE" "$IMAGE"; do - if [ "$SKIP_PULL" = "1" ]; then - docker image inspect "$img" >/dev/null 2>&1 && ok "local image present: $img" || bad "SKIP_PULL=1 but image missing locally: $img" - elif docker pull "$img" >"$WORK/pull_$(echo "$img" | tr '/:' '__').log" 2>&1; then - ok "pulled $img" - elif docker image inspect "$img" >/dev/null 2>&1; then - # Locally built tags (test_locally.sh / docker build) are not on a - # registry; that is fine as long as the image is present. - warn "not pullable but present locally: $img" - else - bad "could not pull $img (see $WORK/pull_*.log)" - fi -done -hr - -# --------------------------------------------------------------------------- # -# 3. GPU passthrough / CPU fallback inside the container -# --------------------------------------------------------------------------- # -bold "3) Container runtime check" -if [ "$GPU_MODE" = "1" ]; then - if docker run --rm "${GPU_FLAG[@]}" "$BASE_IMAGE" python -c \ - "import torch; assert torch.cuda.is_available(); print('torch', torch.__version__, '-', torch.cuda.get_device_name(0))" \ - >"$WORK/gpu_check.log" 2>&1; then - ok "torch.cuda available in-container: $(tail -1 "$WORK/gpu_check.log")" - else - if [ "$NVRT_LISTED" = "1" ]; then - bad "GPU passthrough failed despite a listed nvidia runtime (see $WORK/gpu_check.log) - falling back to CPU mode" - else - warn "--gpus probe failed - docker has no nvidia runtime or CDI spec (install nvidia-container-toolkit); falling back to CPU mode" - fi - tail -5 "$WORK/gpu_check.log" | sed 's/^/ /' - GPU_MODE=0 - fi -fi -if [ "$GPU_MODE" = "0" ]; then - if docker run --rm -e UNSLOTH_ALLOW_CPU=1 "$BASE_IMAGE" python -c \ - "import torch; print('torch', torch.__version__, 'cpu-mode ok')" \ - >"$WORK/cpu_check.log" 2>&1; then - ok "CPU mode boots: $(tail -1 "$WORK/cpu_check.log")" - else - bad "container failed to start even in CPU mode (see $WORK/cpu_check.log)" - tail -5 "$WORK/cpu_check.log" | sed 's/^/ /' - fi -fi -hr - -# --------------------------------------------------------------------------- # -# 4. Training smoke (GPU only): 5 LoRA steps on Llama-3.2-1B 4-bit -# --------------------------------------------------------------------------- # -bold "4) Training smoke" -if [ "$GPU_MODE" = "1" ] && [ "$SKIP_TRAIN" != "1" ]; then - if docker run --rm "${GPU_FLAG[@]}" --ipc=host \ - -v "$HF_CACHE":/workspace/.cache/huggingface \ - ${HF_TOKEN:+-e HF_TOKEN} \ - "$BASE_IMAGE" python /workspace/smoke_test.py >"$WORK/train_smoke.log" 2>&1; then - ok "smoke_test.py: 5 LoRA steps completed" - grep -E '^step|loss' "$WORK/train_smoke.log" | tail -5 | sed 's/^/ /' - else - bad "training smoke failed (see $WORK/train_smoke.log)" - tail -10 "$WORK/train_smoke.log" | sed 's/^/ /' - fi -else - warn "skipped (CPU mode or SKIP_TRAIN=1)" -fi -hr - -# --------------------------------------------------------------------------- # -# 5. GGUF tooling: baked llama.cpp prebuilt -# --------------------------------------------------------------------------- # -bold "5) GGUF tooling (baked llama.cpp)" -if docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 "$BASE_IMAGE" bash -c ' - set -e - test -x "$UNSLOTH_LLAMA_CPP_PATH/llama-quantize" - test -f "$UNSLOTH_LLAMA_CPP_PATH/convert_hf_to_gguf.py" - "$UNSLOTH_LLAMA_CPP_PATH/llama-server" --version 2>&1 | head -2 - cat "$UNSLOTH_LLAMA_CPP_PATH/UNSLOTH_PREBUILT_INFO.json" 2>/dev/null | head -5 - ' >"$WORK/gguf_check.log" 2>&1; then - ok "llama-quantize + llama-server + convert_hf_to_gguf.py present and runnable" - grep -E 'version|asset' "$WORK/gguf_check.log" | head -3 | sed 's/^/ /' -else - bad "baked llama.cpp check failed (see $WORK/gguf_check.log)" - tail -5 "$WORK/gguf_check.log" | sed 's/^/ /' -fi -hr - -# --------------------------------------------------------------------------- # -# 5b. vLLM (GRPO fast_inference=True) -# --------------------------------------------------------------------------- # -bold "5b) vLLM (GRPO fast_inference=True)" -if docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 "$BASE_IMAGE" \ - python -c 'import vllm; print("vllm", vllm.__version__)' \ - >"$WORK/vllm_check.log" 2>&1; then - ok "vllm importable: $(grep -oE 'vllm [0-9][^ ]*' "$WORK/vllm_check.log" | head -1)" -else - IMG_ARCH="$(docker run --rm -e UNSLOTH_SKIP_GPU_CHECK=1 "$BASE_IMAGE" uname -m 2>/dev/null || echo unknown)" - if [ "$IMG_ARCH" = "x86_64" ]; then - bad "vllm missing or broken on x86_64 image (see $WORK/vllm_check.log)" - tail -3 "$WORK/vllm_check.log" | sed 's/^/ /' - else - warn "vllm not available on $IMG_ARCH image; GRPO fast_inference=True unavailable (arm64 wheels are newer, fail-soft at image build)" - fi -fi -hr - -# --------------------------------------------------------------------------- # -# 6. Full image: Studio + JupyterLab boot -# --------------------------------------------------------------------------- # -bold "6) Studio + JupyterLab (full image)" -RUN_ARGS=(-d -p "$PORT_STUDIO":8000 -p "$PORT_JUPYTER":8888) -if [ "$GPU_MODE" = "1" ]; then RUN_ARGS+=("${GPU_FLAG[@]}"); else RUN_ARGS+=(-e UNSLOTH_ALLOW_CPU=1); fi -STUDIO_CID="$(docker run "${RUN_ARGS[@]}" "$IMAGE" 2>"$WORK/studio_run.err")" || STUDIO_CID="" -if [ -z "$STUDIO_CID" ]; then - bad "full image failed to start (see $WORK/studio_run.err)" -else - info "container : ${STUDIO_CID:0:12} (studio http://localhost:$PORT_STUDIO, jupyter http://localhost:$PORT_JUPYTER)" - ok_studio=0; ok_jupyter=0 - for _ in $(seq 1 60); do - if [ "$ok_studio" = 0 ] && curl -fsS "http://localhost:$PORT_STUDIO/api/health" >/dev/null 2>&1; then ok_studio=1; fi - # /login, not /api: a password hash is always configured so /api returns 403. - if [ "$ok_jupyter" = 0 ] && curl -fsS "http://localhost:$PORT_JUPYTER/login" >/dev/null 2>&1; then ok_jupyter=1; fi - [ "$ok_studio" = 1 ] && [ "$ok_jupyter" = 1 ] && break - sleep 5 - done - [ "$ok_studio" = 1 ] && ok "Studio /api/health healthy" || { bad "Studio /api/health never went healthy (docker logs ${STUDIO_CID:0:12})"; docker logs --tail 15 "$STUDIO_CID" 2>&1 | sed 's/^/ /'; } - [ "$ok_jupyter" = 1 ] && ok "JupyterLab /login responding" || bad "JupyterLab /login never responded" -fi -hr - -# --------------------------------------------------------------------------- # -# Summary -# --------------------------------------------------------------------------- # -bold "=== SUMMARY ===" -echo "host : $OS $ARCH wsl=$IS_WSL gpu_mode=$GPU_MODE" -echo "images : $IMAGE / $BASE_IMAGE" -echo "logs : $WORK" -echo "PASS: $PASS_N WARN: $WARN_N FAIL: $FAIL_N" -if [ "$KEEP" = "1" ] && [ -n "$STUDIO_CID" ]; then - echo "container ${STUDIO_CID:0:12} left running (KEEP=1): studio :$PORT_STUDIO jupyter :$PORT_JUPYTER" -fi -if [ "$FAIL_N" -eq 0 ]; then - bold "RESULT: CONFIRMED - the Unsloth Docker images work on this machine." - exit 0 -else - bold "RESULT: $FAIL_N hard failure(s) - paste this whole output back." - exit 1 -fi diff --git a/docker/freeze.sh b/docker/freeze.sh deleted file mode 100755 index 9089ae287c..0000000000 --- a/docker/freeze.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Pull the lockfile out of a built image so the next rebuild can be byte-identical. -# -# ./freeze.sh # extracts to requirements.lock.txt next to Dockerfile -# ./freeze.sh some-tag-or-digest # custom source -# -# To rebuild against the frozen lockfile later, replace the `pip install` lines -# in the Dockerfile with `pip install -r /tmp/requirements.lock.txt --no-deps` -# (mounted via `docker build --build-context lock=./requirements.lock.txt`). -set -euo pipefail - -cd "$(dirname "$0")" - -SRC="${1:-unsloth-blackwell:latest}" -DEST="${2:-./requirements.lock.txt}" - -CID=$(docker create "${SRC}") -trap 'docker rm -f "${CID}" >/dev/null' EXIT - -docker cp "${CID}:/opt/unsloth-venv/requirements.lock.txt" "${DEST}" -echo "Wrote ${DEST}" -echo -echo "Top of lockfile:" -head -20 "${DEST}" -echo -echo "Lines: $(wc -l < "${DEST}")" diff --git a/docker/hf_pull.sh b/docker/hf_pull.sh deleted file mode 100755 index c137494f8f..0000000000 --- a/docker/hf_pull.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# Simulate `docker pull ` against a Hugging Face Hub model repo. -# -# Counterpart to docker/hf_push.sh -- downloads the tar.gz blob from the HF -# repo and `docker load`s it. -# -# Usage: -# bash docker/hf_pull.sh [] [] -# bash docker/hf_pull.sh danielhanchen/unsloth-blackwell-docker unsloth-blackwell-test.tar.gz unsloth-blackwell:test -# -# Requires: docker, pigz (or gzip), hf (or huggingface-cli) authenticated -# (read scope is sufficient for public repos: `hf auth login`). -set -euo pipefail - -REPO="${1:?usage: hf_pull.sh [] []}" -BLOB="${2:-unsloth-blackwell.tar.gz}" -VERIFY="${3:-}" -WORK="${HF_PULL_TMP:-/tmp}" - -command -v docker >/dev/null || { echo "ERROR: docker not on PATH"; exit 1; } - -# Prefer the new `hf` CLI. The old `huggingface-cli` was deprecated in -# huggingface_hub >= 0.27 and silently exits with a deprecation notice -# instead of doing the download, so we treat its presence as a fallback -# only and warn if it's all we have. -if command -v hf >/dev/null 2>&1; then - HF_CMD=(hf download) -elif command -v huggingface-cli >/dev/null 2>&1; then - echo "WARN: 'hf' not found, falling back to 'huggingface-cli' (deprecated)" >&2 - HF_CMD=(huggingface-cli download) -else - echo "ERROR: need 'hf' (pip install -U huggingface_hub) or 'huggingface-cli'"; exit 1 -fi -DECOMPRESSOR=$(command -v pigz || command -v gzip) || { echo "ERROR: need pigz or gzip"; exit 1; } - -DEST="${WORK}/$(basename "${BLOB}")" -echo ">> downloading ${REPO}/${BLOB} -> ${DEST} (via: ${HF_CMD[*]})" -"${HF_CMD[@]}" "${REPO}" "${BLOB}" --repo-type=model --local-dir "${WORK}" -test -s "${DEST}" || { echo "ERROR: download produced no file at ${DEST}"; exit 1; } -ls -lh "${DEST}" - -echo ">> loading into docker (using ${DECOMPRESSOR##*/})" -"${DECOMPRESSOR}" -d -c "${DEST}" | docker load - -if [[ -n "${VERIFY}" ]]; then - if docker image inspect "${VERIFY}" >/dev/null 2>&1; then - echo ">> verified: ${VERIFY} is loaded" - docker image inspect --format 'image_id={{.Id}} size={{.Size}}' "${VERIFY}" - else - echo "WARN: expected tag ${VERIFY} not found after load. docker images:" - docker images - exit 1 - fi -fi diff --git a/docker/hf_push.sh b/docker/hf_push.sh deleted file mode 100755 index b3b94d4909..0000000000 --- a/docker/hf_push.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Simulate `docker push ` against a Hugging Face Hub model repo. -# -# HF Hub doesn't act as an OCI registry for arbitrary images (only Spaces have -# that). So we approximate the push by: -# 1. docker save | pigz -> single tar.gz blob -# 2. huggingface-cli upload to /{tag}.tar.gz -# -# This is good for cross-host testing where you want one canonical place to -# pull from. For the real release, use Docker Hub or GHCR with `docker push`, -# which gives you layer dedup, manifest negotiation, and standard `docker pull` -# UX -- see .github/workflows/docker-publish.yml in this repo. -# -# Usage: -# bash docker/hf_push.sh -# bash docker/hf_push.sh unsloth-blackwell:test danielhanchen/unsloth-blackwell-docker -# -# Requires: docker, pigz (or gzip), hf (or huggingface-cli) authenticated -# with a WRITE-scoped token: `hf auth login`. -set -euo pipefail - -IMAGE="${1:?usage: hf_push.sh }" -REPO="${2:?usage: hf_push.sh }" -TAG="${IMAGE##*:}" -NAME="${IMAGE%:*}" -NAME="${NAME##*/}" -BLOB="${NAME}-${TAG}.tar.gz" -WORK="${HF_PUSH_TMP:-/tmp}" - -command -v docker >/dev/null || { echo "ERROR: docker not on PATH"; exit 1; } - -# Prefer the new `hf` CLI. The old `huggingface-cli` was deprecated in -# huggingface_hub >= 0.27 and silently exits with a deprecation notice -# instead of doing the upload. -if command -v hf >/dev/null 2>&1; then - HF_CMD=(hf upload) -elif command -v huggingface-cli >/dev/null 2>&1; then - echo "WARN: 'hf' not found, falling back to 'huggingface-cli' (deprecated)" >&2 - HF_CMD=(huggingface-cli upload) -else - echo "ERROR: need 'hf' (pip install -U huggingface_hub) or 'huggingface-cli'"; exit 1 -fi -COMPRESSOR=$(command -v pigz || command -v gzip) || { echo "ERROR: need pigz or gzip"; exit 1; } - -OUT="${WORK}/${BLOB}" -echo ">> saving ${IMAGE} -> ${OUT} (using ${COMPRESSOR##*/})" -docker save "${IMAGE}" | "${COMPRESSOR}" > "${OUT}" -ls -lh "${OUT}" - -echo ">> uploading to https://huggingface.co/${REPO}/blob/main/${BLOB} (via: ${HF_CMD[*]})" -"${HF_CMD[@]}" "${REPO}" "${OUT}" "${BLOB}" \ - --repo-type=model \ - --commit-message="push ${IMAGE} ($(docker inspect --format '{{.Id}}' "${IMAGE}" | cut -c8-19))" - -echo ">> pushed." -echo "On the consumer side, run:" -echo " bash docker/hf_pull.sh ${REPO} ${BLOB} ${IMAGE}" diff --git a/docker/setup_qemu.sh b/docker/setup_qemu.sh deleted file mode 100755 index 2f46c6d1e5..0000000000 --- a/docker/setup_qemu.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# One-time host setup: register QEMU binfmt handlers so `docker buildx` can -# build images for foreign architectures (e.g. linux/arm64 on an x86_64 host). -# -# After this runs once per host reboot you can do: -# -# docker buildx build --platform linux/arm64 -t unsloth-blackwell:arm64 . -# docker buildx build --platform linux/amd64,linux/arm64 --push -t YOU/img:tag . -# -# Important: QEMU is used at BUILD time only. The resulting arm64 image must -# be RUN on an aarch64 host (e.g. DGX Spark / GB10) -- CUDA does not work under -# runtime emulation. To smoke-test the arm64 image you need an actual arm64 -# GPU machine. -# -# Usage: -# bash docker/setup_qemu.sh -# -# Requires: docker (28+ recommended), docker buildx plugin, root via sudo or -# membership in the `docker` group. No network access to NVIDIA registries -# is needed for this step. -set -euo pipefail - -command -v docker >/dev/null || { echo "ERROR: docker not on PATH"; exit 1; } -docker buildx version >/dev/null 2>&1 || { - echo "ERROR: 'docker buildx' missing. Install:" >&2 - echo " Ubuntu/Debian: sudo apt-get install -y docker-buildx" >&2 - echo " RHEL/Fedora: sudo dnf install -y docker-buildx-plugin" >&2 - exit 1 -} - -ARCH="$(uname -m)" -echo ">> host arch: ${ARCH}" - -# `tonistiigi/binfmt --install all` registers handlers for every supported -# foreign arch; harmless if some are already registered. This is the canonical -# upstream Docker recipe; see https://docs.docker.com/build/building/multi-platform/ -echo ">> registering QEMU binfmt handlers via tonistiigi/binfmt..." -docker run --privileged --rm tonistiigi/binfmt --install all - -# Ensure we have a buildx builder that can target multiple platforms. -# The default 'docker' driver builder is single-platform; we create (or -# reuse) a 'unsloth-multiarch' container-driver builder which is multi-arch. -BUILDER="unsloth-multiarch" -if docker buildx inspect "${BUILDER}" >/dev/null 2>&1; then - echo ">> buildx builder '${BUILDER}' already exists" -else - echo ">> creating buildx builder '${BUILDER}'" - docker buildx create --name "${BUILDER}" --driver docker-container --use -fi -docker buildx use "${BUILDER}" -docker buildx inspect --bootstrap "${BUILDER}" | sed -n '1,12p' - -echo -echo ">> done. Verify with:" -echo " docker buildx ls" -echo " docker buildx inspect ${BUILDER}" -echo -echo ">> cross-arch build example:" -echo " docker buildx build --platform linux/arm64 -t unsloth-blackwell:arm64 docker/" diff --git a/docker/test_locally.sh b/docker/test_locally.sh deleted file mode 100755 index 86fbabadfc..0000000000 --- a/docker/test_locally.sh +++ /dev/null @@ -1,410 +0,0 @@ -#!/usr/bin/env bash -# End-to-end Docker validation for the unsloth-blackwell image. -# -# Runs three blocks: -# 1. Host pre-flight (docker, nvidia-smi, nvidia runtime registered) -# 2. Build the image (no GPU required at build time) -# 3a. Smoke test: 5-step LoRA on Llama-3.2-1B (~1-2 min) -# 3b. Real workload: gpt-oss-20B fine-tuning notebook with max_steps=10 -# (~10 min, needs ~30GB free for the model cache) -# -# Usage: -# bash docker/test_locally.sh # all blocks (native arch) -# bash docker/test_locally.sh --skip-notebook # blocks 1-3a only (fast) -# bash docker/test_locally.sh --skip-build # assume $TAG already built -# bash docker/test_locally.sh --platform arm64 # cross-build for DGX Spark -# # (auto-skips smoke/notebook) -# TAG=my-image:latest bash docker/test_locally.sh -# HF_TOKEN=hf_xxx bash docker/test_locally.sh # for gated models (optional) -# -# All output is teed to $LOG_DIR (default /tmp/unsloth-docker-test/). -# Paste the listed log snippets back if anything fails. -set -uo pipefail - -TAG="${TAG:-unsloth-blackwell:test}" -LOG_DIR="${LOG_DIR:-/tmp/unsloth-docker-test}" -SKIP_BUILD=0 -SKIP_NOTEBOOK=0 -# Platform selector. Empty = let buildx default to the host arch (no -# --platform passed). "amd64" / "arm64" = single-arch cross-build via QEMU -# (requires `bash docker/setup_qemu.sh` to have been run once). -PLATFORM="" - -while [[ $# -gt 0 ]]; do - case "$1" in - --skip-build) SKIP_BUILD=1; shift ;; - --skip-notebook) SKIP_NOTEBOOK=1; shift ;; - --tag) TAG="$2"; shift 2 ;; - --log-dir) LOG_DIR="$2"; shift 2 ;; - --platform) - case "$2" in - amd64|arm64|linux/amd64|linux/arm64) PLATFORM="${2#linux/}" ;; - *) echo "ERROR: --platform must be amd64 or arm64 (got '$2')" >&2; exit 2 ;; - esac - shift 2 - ;; - --help|-h) sed -n '2,22p' "$0"; exit 0 ;; - *) echo "Unknown flag: $1" >&2; exit 2 ;; - esac -done - -# When cross-building, the resulting image cannot be exercised on this host -# (CUDA does not work under QEMU runtime emulation). Auto-skip the GPU blocks -# and warn the user. They can paste back the build log either way to prove -# the wheels resolve + the build-time torch._C._cuda_getArchFlags() assertion -# passes on the foreign arch. -HOST_ARCH="$(uname -m)" -case "${HOST_ARCH}" in - x86_64|amd64) HOST_DOCKER_ARCH="amd64" ;; - aarch64|arm64) HOST_DOCKER_ARCH="arm64" ;; - *) HOST_DOCKER_ARCH="${HOST_ARCH}" ;; -esac -CROSS_ARCH=0 -if [[ -n "${PLATFORM}" && "${PLATFORM}" != "${HOST_DOCKER_ARCH}" ]]; then - CROSS_ARCH=1 -fi - -mkdir -p "$LOG_DIR" - -GREEN='\033[1;32m'; RED='\033[1;31m'; YELLOW='\033[1;33m'; BLUE='\033[1;34m'; NC='\033[0m' -banner() { printf "\n${BLUE}==== %s ====${NC}\n" "$*"; } -ok() { printf "${GREEN}OK${NC} %s\n" "$*"; } -warn() { printf "${YELLOW}WARN${NC} %s\n" "$*"; } -err() { printf "${RED}ERROR${NC} %s\n" "$*" >&2; } -fail() { err "$*"; exit 1; } - -# ============================================================================ -# Block 1: pre-flight -# ============================================================================ -banner "Block 1: host pre-flight" - -command -v docker >/dev/null 2>&1 || fail "docker not found on PATH" -echo " docker: $(docker --version)" - -# Verify we can talk to the docker daemon as the current user -- catches the -# "user not in docker group" case up front, instead of a later buildx -# "permission denied on /var/run/docker.sock" that masquerades as a build failure. -DOCKER_INFO_OUT=$(docker info 2>&1) -DOCKER_INFO_RC=$? -if [[ $DOCKER_INFO_RC -ne 0 ]]; then - err "Cannot talk to the docker daemon as user '$USER'." - cat >&2 </dev/null 2>&1; then - echo " host gpu: $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" - echo " host driver: $(nvidia-smi --query-gpu=driver_version --format=csv,noheader | head -1)" -else - warn "nvidia-smi not on the host -- you may not be able to run --gpus all" -fi - -# This grep only makes sense once we know `docker info` succeeded above. -if echo "$DOCKER_INFO_OUT" | grep -qiE 'Runtimes:.*nvidia'; then - echo " nvidia runtime: registered with docker" -else - warn "docker info does not list 'nvidia' as a runtime" - warn "(on Docker 28+ with CDI this is often a false positive; the real" - warn " test is whether --gpus all works in Block 3a below)" - warn "if --gpus all fails, install nvidia-container-toolkit:" - warn " https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html" - warn " then: sudo systemctl restart docker" -fi -ok "pre-flight done" - -# ============================================================================ -# Block 2: build -# ============================================================================ -if [[ $SKIP_BUILD -eq 1 ]]; then - warn "skipping build (--skip-build); expecting $TAG to exist" -else - banner "Block 2: build $TAG" - - # Find the build context: current dir, docker/ subdir, or clone the PR branch - if [[ -f "Dockerfile" && -f "smoke_test.py" ]]; then - BUILD_CTX="$PWD" - elif [[ -f "docker/Dockerfile" ]]; then - BUILD_CTX="$PWD/docker" - else - BUILD_CTX="/tmp/unsloth-pr/docker" - if [[ ! -d /tmp/unsloth-pr/.git ]]; then - echo " cloning docker-blackwell-build branch..." - if ! git clone --depth 1 -b docker-blackwell-build \ - https://github.com/unslothai/unsloth.git /tmp/unsloth-pr 2>&1 | tail -3; then - fail "could not clone docker-blackwell-build into /tmp/unsloth-pr; refusing to build from stale context" - fi - else - # `set -e` is not active in this script, so a failing pull would - # otherwise be silently masked and we'd build from a stale clone. - # Explicitly fail loudly when the fast-forward refresh cannot run. - if ! git -C /tmp/unsloth-pr pull --ff-only 2>&1 | tail -2; then - fail "git pull --ff-only failed in /tmp/unsloth-pr; refusing to build from stale context (delete /tmp/unsloth-pr to reclone)" - fi - fi - fi - echo " build context: $BUILD_CTX" - - BUILD_LOG="$LOG_DIR/build.log" - echo " log: $BUILD_LOG" - - # The Dockerfile uses BuildKit-only features ('# syntax=docker/dockerfile:1.7' - # and 'RUN ... <<\'PY\'' heredocs). Docker 28 removed the legacy builder - # entirely -- DOCKER_BUILDKIT=1 now delegates to buildx, so without the - # buildx component installed there is no fallback that works. Fail fast - # with install instructions before attempting the build. - if ! docker buildx version >/dev/null 2>&1; then - cat >&2 <<'MSG' - -ERROR: docker buildx is not installed. - -The Dockerfile requires BuildKit (syntax=docker/dockerfile:1.7 + RUN heredocs). -Docker 28 removed the legacy builder, so buildx is required for any build. - -Install buildx, then re-run this script: - - Ubuntu / Debian (apt): - sudo apt-get update && sudo apt-get install -y docker-buildx - - Ubuntu / Debian (Docker's official repo, recommended): - # Follow https://docs.docker.com/engine/install/ubuntu/ -- the docker-ce - # package bundles docker-buildx-plugin and is what most production guides - # assume. The Ubuntu-shipped docker.io package omits buildx. - - RHEL / Fedora (dnf): - sudo dnf install -y docker-buildx-plugin - - Manual install (any distro): - https://github.com/docker/buildx/releases (download into ~/.docker/cli-plugins/) - -Verify with: docker buildx version -MSG - fail "docker buildx required -- install per the message above" - fi - echo " builder: docker buildx ($(docker buildx version | head -1))" - - BUILD_ARGS=( --progress=plain ) - if [[ -n "${PLATFORM}" ]]; then - echo " platform: linux/${PLATFORM}" - BUILD_ARGS+=( --platform "linux/${PLATFORM}" ) - if [[ ${CROSS_ARCH} -eq 1 ]]; then - echo " cross-build: yes (host=${HOST_DOCKER_ARCH}); verifying QEMU binfmt..." - if ! docker run --rm --privileged tonistiigi/binfmt 2>/dev/null \ - | grep -q "\"linux/${PLATFORM}\""; then - cat >&2 <&1 | tee "$BUILD_LOG" - rc=${PIPESTATUS[0]} - if [[ $rc -ne 0 ]]; then - fail "docker build exited $rc -- see $BUILD_LOG" - fi - - # Sanity check the build's own self-test ran and passed - if grep -q "FAIL: missing wheels\|sm_100 (B200/GB200) missing\|sm_120 (RTX 5090) missing on amd64\|no Blackwell consumer SASS" "$BUILD_LOG"; then - fail "build-time sanity check failed -- see $BUILD_LOG" - fi - grep -E "OK: torch 2.11.0|OK: all required wheels|import cleanly on no-GPU host" "$BUILD_LOG" || \ - warn "could not find 'OK:' lines in build log -- did the verification step run?" - ok "built $TAG" -fi - -# When the image we just built (or were told to use) does not match the host -# architecture, the smoke test and notebook blocks would attempt to launch -# foreign-arch user-space under QEMU plus --gpus all -- which is broken by -# design: nvidia-container-toolkit cannot expose a GPU to a QEMU-emulated -# guest, and even if it could, CUDA kernels do not run under user-space CPU -# emulation. Skip those blocks with a loud warning so the user doesn't think -# they're seeing a real validation pass. -if [[ ${CROSS_ARCH} -eq 1 ]]; then - warn "cross-arch build (host=${HOST_DOCKER_ARCH}, image=${PLATFORM})." - warn "skipping smoke test + notebook -- CUDA does not work under QEMU runtime." - warn "to validate end-to-end on linux/${PLATFORM}, transfer the image to an" - warn "actual ${PLATFORM} host (e.g. DGX Spark for arm64) and re-run with --skip-build." - banner "summary" - echo " image: $TAG" - echo " platform: linux/${PLATFORM} (cross-built on ${HOST_DOCKER_ARCH})" - echo " log dir: $LOG_DIR" - echo - [[ $SKIP_BUILD -eq 0 ]] && echo " to paste back for PR validation:" - [[ $SKIP_BUILD -eq 0 ]] && echo " tail -80 $LOG_DIR/build.log" - ok "cross-arch build verified (wheels + arch-flags assertion passed)" - exit 0 -fi - -# ============================================================================ -# Block 3a: smoke test -# ============================================================================ -banner "Block 3a: smoke test (5-step LoRA on Llama-3.2-1B)" -SMOKE_LOG="$LOG_DIR/smoke.log" -echo " log: $SMOKE_LOG" -docker run --rm --gpus all "$TAG" python /workspace/smoke_test.py 2>&1 | tee "$SMOKE_LOG" -rc=${PIPESTATUS[0]} -if [[ $rc -ne 0 ]]; then - fail "smoke test exited $rc -- see $SMOKE_LOG" -fi -if ! grep -q "all checks passed" "$SMOKE_LOG"; then - fail "smoke test did not print 'all checks passed' -- see $SMOKE_LOG" -fi -ok "smoke test passed" - -# ============================================================================ -# Block 3b: gpt-oss-20B fine-tuning notebook -# ============================================================================ -if [[ $SKIP_NOTEBOOK -eq 1 ]]; then - warn "skipping gpt-oss-20B notebook (--skip-notebook)" -else - banner "Block 3b: gpt-oss-20B fine-tuning notebook (10 LoRA steps)" - GPT_LOG="$LOG_DIR/gpt_oss.log" - HOST_RUN_DIR="$LOG_DIR/host" - mkdir -p "$HOST_RUN_DIR" - echo " log: $GPT_LOG" - echo " host dir: $HOST_RUN_DIR" - - cat > "$HOST_RUN_DIR/run_notebook.sh" <<'INNER' -#!/bin/bash -set -e -cd /workspace/host - -echo "=== install triton_kernels (MXFP4 support for unsloth/gpt-oss-20b) ===" -pip install -q 'git+https://github.com/triton-lang/triton.git@0add68262ab0a2e33b84524346cb27cbb2787356#subdirectory=python/triton_kernels' 2>&1 | tail -5 - -echo -echo "=== fetch + convert notebook ===" -# Use nbformat directly. We then post-process to: -# 1. Skip install cells -- the container already has unsloth + deps baked in; -# the notebook's install cell uses Jupyter !shell magic (raw `!pip install -# ...` lines) that nbformat dumps verbatim and Python cannot parse. -# 2. Comment out any stray !cmd / %magic lines in non-install cells. -pip install -q nbformat -# Pin to an immutable commit so this validation script doesn't silently -# change semantics when notebooks/main rolls forward. Bump deliberately -# when the upstream notebook gets a fix you want to verify against. -NB_REPO_REF="${NB_REPO_REF:-efe20c97a5bba3088b25fe068a4b1c98c0cf3a3a}" -curl -fsSL "https://raw.githubusercontent.com/unslothai/notebooks/${NB_REPO_REF}/nb/gpt-oss-(20B)-Fine-tuning.ipynb" -o nb.ipynb -test -s nb.ipynb || { echo "FAIL: nb.ipynb was not downloaded"; exit 1; } -python - <<'PY' -import nbformat, re -nb = nbformat.read('nb.ipynb', as_version=4) -out, skipped = [], 0 -INSTALL_MARKERS = ( - "pip install", "uv pip install", "apt-get install", - "_original_packages", "COLAB_", "importlib.util.find_spec", -) -for c in nb.cells: - if c.cell_type != "code": - continue - src = c.source or "" - if any(m in src for m in INSTALL_MARKERS): - skipped += 1 - first = next((ln for ln in src.splitlines() if ln.strip()), "")[:80] - out.append(f"# (skipped install/setup cell: {first!r})") - out.append("") - continue - for line in src.splitlines(): - stripped = line.lstrip() - if stripped.startswith(("!", "%")): - out.append(f"# (jupyter magic stripped) {line}") - else: - out.append(line) - out.append("") -with open("nb.py", "w") as f: - f.write("\n".join(out) + "\n") -print(f" converted nb.py: {sum(1 for _ in open('nb.py'))} lines, {skipped} install cell(s) skipped") -PY -test -s nb.py || { echo "FAIL: nb.py was not produced"; exit 1; } -# Sanity-check: nb.py must parse as valid Python before we try to run it. -python -c "import ast; ast.parse(open('nb.py').read()); print(' nb.py is valid Python')" - -echo -echo "=== patch nb.py: max_steps 30 -> 10, drop pre-train demo generations ===" -python - <<'PY' -import re -src = open('nb.py').read() -src = src.replace('max_steps = 30', 'max_steps = 10') -src = re.sub( - r'messages = \[\s*\{[\"\']role[\"\']: [\"\']user[\"\'], [\"\']content[\"\']: [\"\']Solve x\^5.*?\n_ = model\.generate.*?streamer = TextStreamer\(tokenizer\)\)\n', - '# (pre-train inference skipped)\n', - src, flags=re.DOTALL, count=3, -) -open('nb.py', 'w').write(src) -print(' patched. max_steps now:', re.search(r'max_steps = (\d+)', src).group(1)) -PY - -echo -echo "=== run gpt-oss-20B fine-tuning ===" -python -u nb.py -INNER - chmod +x "$HOST_RUN_DIR/run_notebook.sh" - - # Only forward HF_TOKEN if the host has one set, so an empty - # `-e HF_TOKEN=` does not shadow whatever is already inside the image. - # Use the dash-only form `-e HF_TOKEN` so the secret value never - # lands in argv (visible via /proc//cmdline to any user on - # the host for the lifetime of the docker CLI process). - HF_ARGS=() - [[ -n "${HF_TOKEN:-}" ]] && HF_ARGS+=(-e HF_TOKEN) - docker run --rm \ - --gpus all \ - --ipc=host \ - --ulimit memlock=-1 \ - --ulimit stack=67108864 \ - -v "$HOST_RUN_DIR:/workspace/host" \ - -v "$HOME/.cache/huggingface:/workspace/.cache/huggingface" \ - ${HF_ARGS[@]+"${HF_ARGS[@]}"} \ - -e HF_HUB_ENABLE_HF_TRANSFER=1 \ - "$TAG" \ - bash /workspace/host/run_notebook.sh 2>&1 | tee "$GPT_LOG" - rc=${PIPESTATUS[0]} - if [[ $rc -ne 0 ]]; then - fail "gpt-oss-20B notebook exited $rc -- see $GPT_LOG" - fi - ok "gpt-oss-20B notebook completed" -fi - -# ============================================================================ -# Summary -# ============================================================================ -banner "summary" -echo " image: $TAG" -echo " log dir: $LOG_DIR" -echo -echo " to paste back for PR validation:" -[[ $SKIP_BUILD -eq 0 ]] && echo " tail -40 $LOG_DIR/build.log" -echo " cat $LOG_DIR/smoke.log" -[[ $SKIP_NOTEBOOK -eq 0 ]] && echo " tail -100 $LOG_DIR/gpt_oss.log" -echo -ok "all blocks completed"