docker: address review round 3 (notebook -r filter, studio zoo ref, pinned notebooks commit)
- unsloth_pip_shim.py: filter protected packages out of a notebook `pip install -r requirements.txt`. The -r value was passed to the real pip unchanged, so torch / transformers / vLLM / nvidia pins inside the file could overwrite the baked cu128 stack or push transformers into the base venv. _filter_requirements_file() applies the same _KEEP / transformers-sidecar rules per line, writes the survivors to a temp file, keeps comments, option lines, nested includes and urls verbatim, and records a pinned transformers version for the sidecar. - install.sh + Dockerfile.studio + docker-publish.yml: forward the resolved unsloth-zoo ref into the Studio build. install.sh --local overlaid unsloth-zoo from git main regardless of the operator-requested or base-image ref, so the full image could run a different zoo than the base. install.sh now honors UNSLOTH_ZOO_REF across all four --local overlays, Dockerfile.studio passes UNSLOTH_STUDIO_ZOO_REF through to it, and the workflow resolves one zoo ref in the prepare job and shares it with both the base and Studio builds. - Dockerfile + docker-publish.yml: pin unslothai/notebooks to one resolved commit. Each arch leg cloned HEAD independently, so the same tag could seed different baked templates and .unsloth_template_commit depending on the pulled platform. The prepare job freezes notebooks to one sha (like the llama.cpp prebuilt tag) and the Dockerfile fetches that single ref at depth 1.
This commit is contained in:
parent
0ebbdbb9cc
commit
d476c7764b
5 changed files with 173 additions and 45 deletions
95
.github/workflows/docker-publish.yml
vendored
95
.github/workflows/docker-publish.yml
vendored
|
|
@ -48,6 +48,10 @@ on:
|
|||
description: 'unslothai/llama.cpp prebuilt release tag to bake (blank = newest)'
|
||||
required: false
|
||||
default: ''
|
||||
notebooks_ref:
|
||||
description: 'unslothai/notebooks git ref to bake (resolved to one commit)'
|
||||
required: false
|
||||
default: 'main'
|
||||
|
||||
env:
|
||||
REGISTRY: docker.io
|
||||
|
|
@ -85,6 +89,12 @@ jobs:
|
|||
contents: read
|
||||
outputs:
|
||||
llama_tag: ${{ steps.llama.outputs.tag }}
|
||||
# One zoo ref + one notebooks commit, resolved here so BOTH arch legs of
|
||||
# the base build (and the Studio build) bake the identical bits. Resolving
|
||||
# them per-leg would let upstream advance between the amd64 and arm64
|
||||
# builds, putting different content under one published tag.
|
||||
zoo_ref: ${{ steps.zoo_ref.outputs.ref }}
|
||||
notebooks_commit: ${{ steps.notebooks.outputs.commit }}
|
||||
steps:
|
||||
- name: Resolve llama.cpp prebuilt tag
|
||||
id: llama
|
||||
|
|
@ -100,6 +110,46 @@ jobs:
|
|||
echo "tag=${TAG:-latest}" >> "$GITHUB_OUTPUT"
|
||||
echo "llama.cpp prebuilt tag: ${TAG:-latest}"
|
||||
|
||||
# Mirror the unsloth tag into the zoo ONLY when that tag actually exists
|
||||
# there. unsloth's v* tags are Studio releases the zoo never cuts (the zoo
|
||||
# repo currently has no tags at all), so blindly mirroring github.ref_name
|
||||
# made every tag publish fail inside the Dockerfile's zoo install. Resolved
|
||||
# once here and forwarded to the base build AND the Studio build, so the
|
||||
# full image's Studio venv runs the same zoo as the base image.
|
||||
- name: Resolve unsloth-zoo ref
|
||||
id: zoo_ref
|
||||
run: |
|
||||
REF="${{ github.event.inputs.unsloth_zoo_ref }}"
|
||||
if [ -z "$REF" ] && [ "${{ startsWith(github.ref, 'refs/tags/') }}" = "true" ]; then
|
||||
if git ls-remote --exit-code --tags https://github.com/unslothai/unsloth-zoo \
|
||||
"refs/tags/${{ github.ref_name }}" >/dev/null 2>&1; then
|
||||
REF="${{ github.ref_name }}"
|
||||
fi
|
||||
fi
|
||||
echo "ref=${REF:-main}" >> "$GITHUB_OUTPUT"
|
||||
echo "unsloth-zoo ref: ${REF:-main}"
|
||||
|
||||
# Freeze unslothai/notebooks to ONE concrete commit so both arch legs (and
|
||||
# release reruns) bake the identical baked-notebook templates and
|
||||
# .unsloth_template_commit, even if upstream advances mid-build. A 40-char
|
||||
# sha input is already frozen; a branch/tag (default main) is resolved to
|
||||
# its current sha via ls-remote, falling back to the bare ref on a lookup
|
||||
# miss so the Dockerfile can still fetch it by name.
|
||||
- name: Resolve unsloth/notebooks commit
|
||||
id: notebooks
|
||||
env:
|
||||
INPUT_REF: ${{ github.event.inputs.notebooks_ref }}
|
||||
run: |
|
||||
REF="${INPUT_REF:-main}"
|
||||
if printf '%s' "$REF" | grep -Eq '^[0-9a-f]{40}$'; then
|
||||
SHA="$REF"
|
||||
else
|
||||
SHA="$(git ls-remote https://github.com/unslothai/notebooks "$REF" | awk 'NR==1{print $1}')"
|
||||
[ -n "$SHA" ] || SHA="$REF"
|
||||
fi
|
||||
echo "commit=${SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "notebooks commit: ${SHA}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-arch build. The matrix fans out two parallel jobs on the matching
|
||||
# native runner. Each pushes a single-arch image *by digest* (no human-
|
||||
|
|
@ -164,24 +214,6 @@ jobs:
|
|||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
|
||||
# Mirror the unsloth tag into the zoo ONLY when that tag actually
|
||||
# exists there. unsloth's v* tags are Studio releases the zoo never
|
||||
# cuts (the zoo repo currently has no tags at all), so blindly
|
||||
# mirroring github.ref_name made every tag publish fail inside the
|
||||
# Dockerfile's zoo install.
|
||||
- name: Resolve unsloth-zoo ref
|
||||
id: zoo_ref
|
||||
run: |
|
||||
REF="${{ github.event.inputs.unsloth_zoo_ref }}"
|
||||
if [ -z "$REF" ] && [ "${{ startsWith(github.ref, 'refs/tags/') }}" = "true" ]; then
|
||||
if git ls-remote --exit-code --tags https://github.com/unslothai/unsloth-zoo \
|
||||
"refs/tags/${{ github.ref_name }}" >/dev/null 2>&1; then
|
||||
REF="${{ github.ref_name }}"
|
||||
fi
|
||||
fi
|
||||
echo "ref=${REF:-main}" >> "$GITHUB_OUTPUT"
|
||||
echo "unsloth-zoo ref: ${REF:-main}"
|
||||
|
||||
- name: Build and push (per-arch by digest)
|
||||
id: build
|
||||
uses: docker/build-push-action@v6
|
||||
|
|
@ -202,18 +234,21 @@ jobs:
|
|||
# pushes bake the tag's source ref (e.g. v1.2.3) so the published
|
||||
# image actually contains that release; branch + scheduled runs bake
|
||||
# the triggering commit SHA; any other event falls back to main.
|
||||
# UNSLOTH_ZOO_REF (from the resolve step above): explicit dispatch
|
||||
# input, else the pushed tag IF the zoo repo has it, else main -- a
|
||||
# branch SHA does not exist in the zoo repo.
|
||||
# LLAMA_PREBUILT_TAG (from the prepare job): one concrete tag shared
|
||||
# by both arch legs so the published manifest is reproducible.
|
||||
# UNSLOTH_ZOO_REF (from the prepare job): explicit dispatch input,
|
||||
# else the pushed tag IF the zoo repo has it, else main -- a branch
|
||||
# SHA does not exist in the zoo repo. Resolved once in `prepare` and
|
||||
# shared with the Studio build so both venvs run the same zoo.
|
||||
# LLAMA_PREBUILT_TAG / UNSLOTH_NOTEBOOKS_REF (from the prepare job):
|
||||
# one concrete tag / commit shared by both arch legs so the
|
||||
# published manifest is byte-reproducible across platforms.
|
||||
build-args: |
|
||||
CUDA_VERSION=12.8.1
|
||||
UBUNTU_VERSION=24.04
|
||||
PYTHON_VERSION=3.12
|
||||
UNSLOTH_REF=${{ github.event.inputs.unsloth_ref || (startsWith(github.ref, 'refs/tags/') && github.ref_name) || github.sha || 'main' }}
|
||||
UNSLOTH_ZOO_REF=${{ steps.zoo_ref.outputs.ref }}
|
||||
UNSLOTH_ZOO_REF=${{ needs.prepare.outputs.zoo_ref }}
|
||||
LLAMA_PREBUILT_TAG=${{ needs.prepare.outputs.llama_tag }}
|
||||
UNSLOTH_NOTEBOOKS_REF=${{ needs.prepare.outputs.notebooks_commit }}
|
||||
|
||||
# Stash the per-arch digest as an artifact for the merge job to pick up.
|
||||
# Filenames need to be unique across the matrix; `platform` contains a
|
||||
|
|
@ -319,7 +354,9 @@ jobs:
|
|||
# that is the long pole, hence the larger timeout.
|
||||
# ---------------------------------------------------------------------------
|
||||
build-studio:
|
||||
needs: merge
|
||||
# `merge` for the freshly-published base manifest digest; `prepare` for the
|
||||
# one resolved zoo ref (job outputs only flow through direct `needs`).
|
||||
needs: [prepare, merge]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
@ -382,11 +419,15 @@ jobs:
|
|||
cache-to: type=gha,scope=studio-${{ matrix.platform }},mode=min
|
||||
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
|
||||
# UNSLOTH_STUDIO_REF mirrors the base job's UNSLOTH_REF resolution so the
|
||||
# Studio tree matches the unsloth baked into the base venv. (Prose stays
|
||||
# out of build-args -- forwarded lines must be KEY=VALUE only.)
|
||||
# Studio tree matches the unsloth baked into the base venv.
|
||||
# UNSLOTH_STUDIO_ZOO_REF is the SAME resolved zoo ref the base build
|
||||
# baked, so install.sh --local overlays the Studio venv with that zoo
|
||||
# instead of always tracking main. (Prose stays out of build-args --
|
||||
# forwarded lines must be KEY=VALUE only.)
|
||||
build-args: |
|
||||
BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.merge.outputs.digest }}
|
||||
UNSLOTH_STUDIO_REF=${{ github.event.inputs.unsloth_ref || (startsWith(github.ref, 'refs/tags/') && github.ref_name) || github.sha || 'main' }}
|
||||
UNSLOTH_STUDIO_ZOO_REF=${{ needs.prepare.outputs.zoo_ref }}
|
||||
|
||||
- name: Export digest
|
||||
run: |
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue