diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 943d973c16..c62ff97421 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -119,7 +119,12 @@ jobs: CUDA_VERSION=12.8.1 UBUNTU_VERSION=24.04 PYTHON_VERSION=3.12 - UNSLOTH_REF=${{ github.event.inputs.unsloth_ref || 'main' }} + # Workflow-dispatch: honour the explicit input. Tag pushes: + # bake the tag's source ref (e.g. v1.2.3) so the published + # tag image actually contains that release. Branch pushes and + # scheduled runs: bake the triggering commit SHA. Falls back + # to `main` for any other event class. + UNSLOTH_REF=${{ github.event.inputs.unsloth_ref || (startsWith(github.ref, 'refs/tags/') && github.ref_name) || github.sha || 'main' }} UNSLOTH_ZOO_REF=${{ github.event.inputs.unsloth_zoo_ref || 'main' }} # Stash the per-arch digest as an artifact for the merge job to pick up. @@ -203,9 +208,30 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 + + # Re-compute the tag list deterministically from the same metadata-action + # config the merge job used, so tag/schedule/SHA runs pull the image + # they just published instead of an unrelated `:latest` from a prior run. + - name: Resolve published tag + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=tag + type=schedule,pattern=nightly + type=sha,prefix=sha-,format=short + - name: Pull and smoke-test run: | - docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - docker run --rm --gpus all \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \ - python /workspace/smoke_test.py + # Use the first tag from the metadata output -- that is the image we + # just published. Falls back to :latest only when the metadata is + # empty (defensive; should not happen on default-branch runs). + TAG="$(jq -r '.tags[0] // ""' <<<"$DOCKER_METADATA_OUTPUT_JSON")" + if [ -z "$TAG" ]; then + TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" + fi + echo "smoke-testing $TAG" + docker pull "$TAG" + docker run --rm --gpus all "$TAG" python /workspace/smoke_test.py diff --git a/async_task_outputs/async_task_output_1rtbzl.md b/async_task_outputs/async_task_output_1rtbzl.md new file mode 100644 index 0000000000..5a09648069 --- /dev/null +++ b/async_task_outputs/async_task_output_1rtbzl.md @@ -0,0 +1 @@ +- Docker GPU-free build rationale done \ No newline at end of file diff --git a/async_task_outputs/async_task_output_1ud2z5.md b/async_task_outputs/async_task_output_1ud2z5.md new file mode 100644 index 0000000000..9da22f81e7 --- /dev/null +++ b/async_task_outputs/async_task_output_1ud2z5.md @@ -0,0 +1 @@ +- Docker ok; run `bash docker/test_locally.sh --skip-notebook` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_1vzwo0.md b/async_task_outputs/async_task_output_1vzwo0.md new file mode 100644 index 0000000000..9de146d919 --- /dev/null +++ b/async_task_outputs/async_task_output_1vzwo0.md @@ -0,0 +1 @@ +- PR `#5748` pushed commit `58693c4c` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_215bp0.md b/async_task_outputs/async_task_output_215bp0.md new file mode 100644 index 0000000000..34401cdd97 --- /dev/null +++ b/async_task_outputs/async_task_output_215bp0.md @@ -0,0 +1 @@ +- `externally-managed-environment` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_31h69u.md b/async_task_outputs/async_task_output_31h69u.md new file mode 100644 index 0000000000..54567aac0b --- /dev/null +++ b/async_task_outputs/async_task_output_31h69u.md @@ -0,0 +1 @@ +- PR `#5748`: added/pushed script. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_3aetsv.md b/async_task_outputs/async_task_output_3aetsv.md new file mode 100644 index 0000000000..88506f943f --- /dev/null +++ b/async_task_outputs/async_task_output_3aetsv.md @@ -0,0 +1 @@ +- Buildx required; edited/pushed `56d2701a` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_3k07zq.md b/async_task_outputs/async_task_output_3k07zq.md new file mode 100644 index 0000000000..dcbd870300 --- /dev/null +++ b/async_task_outputs/async_task_output_3k07zq.md @@ -0,0 +1 @@ +- PR `unslothai/unsloth`: `sm_120` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_3xv17f.md b/async_task_outputs/async_task_output_3xv17f.md new file mode 100644 index 0000000000..93d3f70065 --- /dev/null +++ b/async_task_outputs/async_task_output_3xv17f.md @@ -0,0 +1 @@ +- Build OK; smoke `ImportError` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_5etrdc.md b/async_task_outputs/async_task_output_5etrdc.md new file mode 100644 index 0000000000..17fe278320 --- /dev/null +++ b/async_task_outputs/async_task_output_5etrdc.md @@ -0,0 +1 @@ +- Pushed `f7b34793`; pending retry. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_5ih1fr.md b/async_task_outputs/async_task_output_5ih1fr.md new file mode 100644 index 0000000000..cf0af94592 --- /dev/null +++ b/async_task_outputs/async_task_output_5ih1fr.md @@ -0,0 +1 @@ +- Pushed `23a5b431`; pending retry. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_778ozt.md b/async_task_outputs/async_task_output_778ozt.md new file mode 100644 index 0000000000..51e4ab198f --- /dev/null +++ b/async_task_outputs/async_task_output_778ozt.md @@ -0,0 +1 @@ +- `Failed to find C compiler`; exit 1 \ No newline at end of file diff --git a/async_task_outputs/async_task_output_8cu1bq.md b/async_task_outputs/async_task_output_8cu1bq.md new file mode 100644 index 0000000000..8e6e8c034d --- /dev/null +++ b/async_task_outputs/async_task_output_8cu1bq.md @@ -0,0 +1 @@ +- Asked: `test on other machines` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_8rdvq1.md b/async_task_outputs/async_task_output_8rdvq1.md new file mode 100644 index 0000000000..033bf7cb72 --- /dev/null +++ b/async_task_outputs/async_task_output_8rdvq1.md @@ -0,0 +1 @@ +- Asked: `gzip` multiprocessing? slow \ No newline at end of file diff --git a/async_task_outputs/async_task_output_924viw.md b/async_task_outputs/async_task_output_924viw.md new file mode 100644 index 0000000000..21d65c3a9f --- /dev/null +++ b/async_task_outputs/async_task_output_924viw.md @@ -0,0 +1 @@ +- Asked: `So what did you do to make the docker work?` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_98ywdm.md b/async_task_outputs/async_task_output_98ywdm.md new file mode 100644 index 0000000000..0a2d17421f --- /dev/null +++ b/async_task_outputs/async_task_output_98ywdm.md @@ -0,0 +1 @@ +- Wants executable `sh`-like script \ No newline at end of file diff --git a/async_task_outputs/async_task_output_9gyz0z.md b/async_task_outputs/async_task_output_9gyz0z.md new file mode 100644 index 0000000000..981816e803 --- /dev/null +++ b/async_task_outputs/async_task_output_9gyz0z.md @@ -0,0 +1 @@ +- Chose apt `docker-buildx`; run `bash docker/test_locally.sh --skip-notebook` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_9stc7n.md b/async_task_outputs/async_task_output_9stc7n.md new file mode 100644 index 0000000000..e15d6d6eee --- /dev/null +++ b/async_task_outputs/async_task_output_9stc7n.md @@ -0,0 +1 @@ +- Recommended `pigz`; gzip-compatible. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_a6uutw.md b/async_task_outputs/async_task_output_a6uutw.md new file mode 100644 index 0000000000..1465964a6e --- /dev/null +++ b/async_task_outputs/async_task_output_a6uutw.md @@ -0,0 +1 @@ +- Ran `docker info`; Docker works \ No newline at end of file diff --git a/async_task_outputs/async_task_output_a72mjm.md b/async_task_outputs/async_task_output_a72mjm.md new file mode 100644 index 0000000000..95d9e2f845 --- /dev/null +++ b/async_task_outputs/async_task_output_a72mjm.md @@ -0,0 +1,2 @@ +- `dde5170e` pushed +- GPU tests pending \ No newline at end of file diff --git a/async_task_outputs/async_task_output_af1yr3.md b/async_task_outputs/async_task_output_af1yr3.md new file mode 100644 index 0000000000..e0402a5840 --- /dev/null +++ b/async_task_outputs/async_task_output_af1yr3.md @@ -0,0 +1 @@ +- Asked why RTX 6000/5090 needed \ No newline at end of file diff --git a/async_task_outputs/async_task_output_b6xdx8.md b/async_task_outputs/async_task_output_b6xdx8.md new file mode 100644 index 0000000000..2c03db6ca4 --- /dev/null +++ b/async_task_outputs/async_task_output_b6xdx8.md @@ -0,0 +1 @@ +- `docker build --progress` unsupported; build failed exit `125` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_c76tz2.md b/async_task_outputs/async_task_output_c76tz2.md new file mode 100644 index 0000000000..d7770e951d --- /dev/null +++ b/async_task_outputs/async_task_output_c76tz2.md @@ -0,0 +1 @@ +- Fixed Triton JIT via `1cdc5f17` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_d9vccx.md b/async_task_outputs/async_task_output_d9vccx.md new file mode 100644 index 0000000000..d6dfc8c67f --- /dev/null +++ b/async_task_outputs/async_task_output_d9vccx.md @@ -0,0 +1 @@ +- Pending: upload `/tmp/unsloth-blackwell.tar.gz` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_dmy1yd.md b/async_task_outputs/async_task_output_dmy1yd.md new file mode 100644 index 0000000000..20da581e1c --- /dev/null +++ b/async_task_outputs/async_task_output_dmy1yd.md @@ -0,0 +1 @@ +- Gave docker build/run commands; pending output \ No newline at end of file diff --git a/async_task_outputs/async_task_output_e7m19a.md b/async_task_outputs/async_task_output_e7m19a.md new file mode 100644 index 0000000000..6c8813dda6 --- /dev/null +++ b/async_task_outputs/async_task_output_e7m19a.md @@ -0,0 +1 @@ +- Asked `docker` GPU for `unsloth` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_eu96wu.md b/async_task_outputs/async_task_output_eu96wu.md new file mode 100644 index 0000000000..62bd87bc0c --- /dev/null +++ b/async_task_outputs/async_task_output_eu96wu.md @@ -0,0 +1 @@ +- `56d2701a`: buildx fix OK; done \ No newline at end of file diff --git a/async_task_outputs/async_task_output_f3n8gk.md b/async_task_outputs/async_task_output_f3n8gk.md new file mode 100644 index 0000000000..73a0c4e668 --- /dev/null +++ b/async_task_outputs/async_task_output_f3n8gk.md @@ -0,0 +1 @@ +- Asked `So what should we try next` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_hfha2k.md b/async_task_outputs/async_task_output_hfha2k.md new file mode 100644 index 0000000000..e516923b26 --- /dev/null +++ b/async_task_outputs/async_task_output_hfha2k.md @@ -0,0 +1 @@ +- Build fails: `docker buildx` missing \ No newline at end of file diff --git a/async_task_outputs/async_task_output_hni3eq.md b/async_task_outputs/async_task_output_hni3eq.md new file mode 100644 index 0000000000..0baecb40f8 --- /dev/null +++ b/async_task_outputs/async_task_output_hni3eq.md @@ -0,0 +1 @@ +- fixed Dockerfile PEP668; pushed `fd55ed0a` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_j4howx.md b/async_task_outputs/async_task_output_j4howx.md new file mode 100644 index 0000000000..e1580c311d --- /dev/null +++ b/async_task_outputs/async_task_output_j4howx.md @@ -0,0 +1 @@ +- `latest docker`; vllm web search \ No newline at end of file diff --git a/async_task_outputs/async_task_output_j5k6f1.md b/async_task_outputs/async_task_output_j5k6f1.md new file mode 100644 index 0000000000..2257131b14 --- /dev/null +++ b/async_task_outputs/async_task_output_j5k6f1.md @@ -0,0 +1 @@ +- `Ok you upload to danielhanchen` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_jogf0i.md b/async_task_outputs/async_task_output_jogf0i.md new file mode 100644 index 0000000000..3d1b668083 --- /dev/null +++ b/async_task_outputs/async_task_output_jogf0i.md @@ -0,0 +1 @@ +- pushed `4bfb4b89` add `docker/hf_*.sh` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_k4wy4g.md b/async_task_outputs/async_task_output_k4wy4g.md new file mode 100644 index 0000000000..c98d472fb9 --- /dev/null +++ b/async_task_outputs/async_task_output_k4wy4g.md @@ -0,0 +1 @@ +- HF repo made; upload tar pending. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_lls2b3.md b/async_task_outputs/async_task_output_lls2b3.md new file mode 100644 index 0000000000..445c1f85ba --- /dev/null +++ b/async_task_outputs/async_task_output_lls2b3.md @@ -0,0 +1 @@ +Continue the user’s latest directive: make the Docker setup work for DGX Spark via emulation. The WebSearch tool for Docker buildx multi-platform/QEMU CUDA build context is already done, so do not re-run it. Next, inspect the search result context and continue researching current linux/arm64 CUDA/PyTorch/xformers/bitsandbytes/flash-attn wheel availability, then update the Docker/build/test/publish files as needed for a build-time QEMU arm64 path. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_ltr06m.md b/async_task_outputs/async_task_output_ltr06m.md new file mode 100644 index 0000000000..b9810162c5 --- /dev/null +++ b/async_task_outputs/async_task_output_ltr06m.md @@ -0,0 +1 @@ +- Need support SM 7.5..12.1 \ No newline at end of file diff --git a/async_task_outputs/async_task_output_nx17q0.md b/async_task_outputs/async_task_output_nx17q0.md new file mode 100644 index 0000000000..0c6ae3f13e --- /dev/null +++ b/async_task_outputs/async_task_output_nx17q0.md @@ -0,0 +1 @@ +- LoRA train OK; all blocks completed \ No newline at end of file diff --git a/async_task_outputs/async_task_output_oo1b5k.md b/async_task_outputs/async_task_output_oo1b5k.md new file mode 100644 index 0000000000..3aecea1529 --- /dev/null +++ b/async_task_outputs/async_task_output_oo1b5k.md @@ -0,0 +1 @@ +- User said `Yes try it out` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_ptu313.md b/async_task_outputs/async_task_output_ptu313.md new file mode 100644 index 0000000000..9c513c38b7 --- /dev/null +++ b/async_task_outputs/async_task_output_ptu313.md @@ -0,0 +1 @@ +- PR `#5748` opened; push used scoped token \ No newline at end of file diff --git a/async_task_outputs/async_task_output_q9q3pp.md b/async_task_outputs/async_task_output_q9q3pp.md new file mode 100644 index 0000000000..c14da2490a --- /dev/null +++ b/async_task_outputs/async_task_output_q9q3pp.md @@ -0,0 +1 @@ +- Docker push via `huggingface_hub` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_qe4w44.md b/async_task_outputs/async_task_output_qe4w44.md new file mode 100644 index 0000000000..b90e02e5df --- /dev/null +++ b/async_task_outputs/async_task_output_qe4w44.md @@ -0,0 +1 @@ +- Ran `docker save`; completed \ No newline at end of file diff --git a/async_task_outputs/async_task_output_qp3v3e.md b/async_task_outputs/async_task_output_qp3v3e.md new file mode 100644 index 0000000000..cae1c85314 --- /dev/null +++ b/async_task_outputs/async_task_output_qp3v3e.md @@ -0,0 +1 @@ +- Docker build needn't GPU; sm_120 only test \ No newline at end of file diff --git a/async_task_outputs/async_task_output_r3h3hd.md b/async_task_outputs/async_task_output_r3h3hd.md new file mode 100644 index 0000000000..713168d8d5 --- /dev/null +++ b/async_task_outputs/async_task_output_r3h3hd.md @@ -0,0 +1 @@ +Continue the latest directive: make the Docker image work for DGX Spark via build-time emulation, targeting a proper linux/arm64 image rather than CUDA runtime emulation. The assistant had started checking current multi-platform/QEMU and arm64 CUDA/PyTorch wheel availability; one WebSearch result for Docker buildx multi-platform builds has returned, so do not re-run that completed search. Next action is to continue the remaining live research from the in-flight WebSearch context, especially whether PyTorch/CUDA, xformers, bitsandbytes, flash-attn, and Unsloth dependencies have usable arm64 wheels or need conditional Dockerfile handling. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_rccvqp.md b/async_task_outputs/async_task_output_rccvqp.md new file mode 100644 index 0000000000..de356eb57f --- /dev/null +++ b/async_task_outputs/async_task_output_rccvqp.md @@ -0,0 +1 @@ +- PR #5748 smoke passed; comment posted. \ No newline at end of file diff --git a/async_task_outputs/async_task_output_re5vqk.md b/async_task_outputs/async_task_output_re5vqk.md new file mode 100644 index 0000000000..f22fd31fa0 --- /dev/null +++ b/async_task_outputs/async_task_output_re5vqk.md @@ -0,0 +1 @@ +- Validated; fixed `Dockerfile`; pending none \ No newline at end of file diff --git a/async_task_outputs/async_task_output_s3bkm5.md b/async_task_outputs/async_task_output_s3bkm5.md new file mode 100644 index 0000000000..8ba978d6d5 --- /dev/null +++ b/async_task_outputs/async_task_output_s3bkm5.md @@ -0,0 +1 @@ +- PENDING: test Docker notebook \ No newline at end of file diff --git a/async_task_outputs/async_task_output_tqe3ko.md b/async_task_outputs/async_task_output_tqe3ko.md new file mode 100644 index 0000000000..4a420f3aee --- /dev/null +++ b/async_task_outputs/async_task_output_tqe3ko.md @@ -0,0 +1 @@ +- Pushed `00cbc825`; retry `bash docker/test_locally.sh --skip-notebook` \ No newline at end of file diff --git a/async_task_outputs/async_task_output_uudy1y.md b/async_task_outputs/async_task_output_uudy1y.md new file mode 100644 index 0000000000..f10ed4f59a --- /dev/null +++ b/async_task_outputs/async_task_output_uudy1y.md @@ -0,0 +1 @@ +- `--skip-notebook`; Docker smoke OK \ No newline at end of file diff --git a/async_task_outputs/async_task_output_wcgi8q.md b/async_task_outputs/async_task_output_wcgi8q.md new file mode 100644 index 0000000000..7deb745c20 --- /dev/null +++ b/async_task_outputs/async_task_output_wcgi8q.md @@ -0,0 +1 @@ +- Asked if Blackwell image supports GPUs \ No newline at end of file diff --git a/async_task_outputs/async_task_output_xgyhvf.md b/async_task_outputs/async_task_output_xgyhvf.md new file mode 100644 index 0000000000..d6b8b89daa --- /dev/null +++ b/async_task_outputs/async_task_output_xgyhvf.md @@ -0,0 +1 @@ +- Installed `docker-buildx`; build failed Docker socket perm \ No newline at end of file diff --git a/async_task_outputs/async_task_output_yt3d7j.md b/async_task_outputs/async_task_output_yt3d7j.md new file mode 100644 index 0000000000..5528adbf01 --- /dev/null +++ b/async_task_outputs/async_task_output_yt3d7j.md @@ -0,0 +1 @@ +- Answered GPU support matrix; DGX pending \ No newline at end of file diff --git a/async_task_outputs/async_task_output_zsqsui.md b/async_task_outputs/async_task_output_zsqsui.md new file mode 100644 index 0000000000..dadef75361 --- /dev/null +++ b/async_task_outputs/async_task_output_zsqsui.md @@ -0,0 +1 @@ +- B200 OK; pending HF upload cmds \ No newline at end of file diff --git a/docker/smoke_test.py b/docker/smoke_test.py index a2ca70225f..2c1a6442f1 100644 --- a/docker/smoke_test.py +++ b/docker/smoke_test.py @@ -63,9 +63,15 @@ def check_imports() -> None: import unsloth_zoo print(f"unsloth_zoo {unsloth_zoo.__version__}") - import xformers - - print(f"xformers {xformers.__version__}") + # xformers is not built for aarch64 cu128 as of this writing; the arm64 + # variant of this image installs unsloth with `[huggingface]` extras + # which omits it. Treat the import as best-effort so the same script + # smoke-tests both arches. + try: + import xformers + print(f"xformers {xformers.__version__}") + except ImportError: + print("xformers (missing -- expected on arm64 [huggingface] extras)") import bitsandbytes as bnb print(f"bnb {bnb.__version__}") diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py index ecad52f8ba..4a77b833a4 100644 --- a/unsloth/_gpu_init.py +++ b/unsloth/_gpu_init.py @@ -138,6 +138,21 @@ except ModuleNotFoundError: except: raise +# Re-assert the single-compile-worker policy after unsloth_zoo has had a +# chance to run its patch_torch_compile (which historically popped +# TORCHINDUCTOR_COMPILE_THREADS in non-debug mode). Force the Inductor +# config value directly so the Docker --gpus '"device=N"' subprocess-pool +# bug is fixed even when the installed unsloth_zoo predates the +# corresponding zoo-side patch. No-op when the user opted out. +if os.environ.get("UNSLOTH_FORCE_SINGLE_COMPILE_WORKER", "0") == "1": + try: + torch._inductor.config.compile_threads = 1 + except Exception: + pass + # Re-populate the env var so determine_compile_threads in the zoo + # options dict also sees it; cheap and forward-compatible. + os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1" + from unsloth_zoo.device_type import ( is_hip, get_device_type,