diff --git a/docker/Dockerfile b/docker/Dockerfile index 77c09061ad..1dca2799e1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -642,15 +642,16 @@ RUN mkdir -p ${HF_HOME} ${TRITON_CACHE_DIR} # * unsloth-run: headless `unsloth-run ` that auto-picks the # sidecar and executes every cell -- the robust driven path. # --------------------------------------------------------------------------- -COPY unsloth_nb_compat.py unsloth_pip_shim.py unsloth_ipython_startup.py unsloth_run.py unsloth_sync_notebooks.sh /opt/unsloth-nb/ +COPY unsloth_nb_compat.py unsloth_pip_shim.py unsloth_ipython_startup.py unsloth_run.py unsloth_sync_notebooks.sh unsloth_nb_content_sig.py /opt/unsloth-nb/ RUN set -eux \ && SP=/opt/unsloth-venv/lib/python${PYTHON_VERSION}/site-packages \ && cp /opt/unsloth-nb/unsloth_nb_compat.py "$SP/unsloth_nb_compat.py" \ - && chmod +x /opt/unsloth-nb/unsloth_pip_shim.py /opt/unsloth-nb/unsloth_run.py /opt/unsloth-nb/unsloth_sync_notebooks.sh \ + && chmod +x /opt/unsloth-nb/unsloth_pip_shim.py /opt/unsloth-nb/unsloth_run.py /opt/unsloth-nb/unsloth_sync_notebooks.sh /opt/unsloth-nb/unsloth_nb_content_sig.py \ && mkdir -p /opt/unsloth-nb/bin \ && for t in pip pip3 uv; do ln -sf /opt/unsloth-nb/unsloth_pip_shim.py /opt/unsloth-nb/bin/$t; done \ && ln -sf /opt/unsloth-nb/unsloth_run.py /usr/local/bin/unsloth-run \ && ln -sf /opt/unsloth-nb/unsloth_sync_notebooks.sh /usr/local/bin/unsloth-sync-notebooks \ + && ln -sf /opt/unsloth-nb/unsloth_nb_content_sig.py /usr/local/bin/unsloth-nb-content-sig \ && mkdir -p /root/.ipython/profile_default/startup \ && cp /opt/unsloth-nb/unsloth_ipython_startup.py /root/.ipython/profile_default/startup/00-unsloth-nb.py \ && /opt/unsloth-venv/bin/python -c "import sys, glob; sys.path.insert(0, '$SP'); import unsloth_nb_compat; print('nb-compat OK; baked sidecars:', sorted(glob.glob('/opt/unsloth-venv/tf-sidecars/t_*')))" @@ -661,8 +662,11 @@ ENV PATH=/opt/unsloth-nb/bin:${PATH} # present (no git clone or wget needed). Baked here as a READ-ONLY template # (~206MB, .git stripped); on boot the entrypoint copies it to # /workspace/unsloth-notebooks and best-effort refreshes from GitHub when -# upstream has advanced, never overwriting a notebook the user has edited (see -# unsloth_sync_notebooks.sh). Inherited as-is by the studio image (FROM base). +# upstream has advanced. It never overwrites a notebook the user has touched, +# and for untouched notebooks it skips the rewrite when only the install header +# / announcements / footer moved upstream (the tutorial body is unchanged) -- +# see unsloth_sync_notebooks.sh + unsloth_nb_content_sig.py. Inherited as-is by +# the studio image (FROM base). RUN set -eux \ && git clone --depth 1 https://github.com/unslothai/notebooks /opt/unsloth-notebooks \ && git -C /opt/unsloth-notebooks rev-parse HEAD > /opt/unsloth-notebooks/.unsloth_template_commit \ diff --git a/docker/unsloth_nb_content_sig.py b/docker/unsloth_nb_content_sig.py new file mode 100644 index 0000000000..832eef12a9 --- /dev/null +++ b/docker/unsloth_nb_content_sig.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Compare the *content* of two Unsloth notebooks, ignoring the auto-generated +# top/bottom boilerplate that update_all_notebooks.py stamps on every notebook. +# +# Every generated notebook has the same shape: +# - a "top" of boilerplate: the "To run this, press Runtime" announcement, the +# "### News" / Unsloth Studio announcement cells, and the %%capture install +# cell. These churn constantly (new pip pins, new announcements, new links). +# - the "middle": the actual tutorial (data prep, train, inference, save). +# - a "bottom": the "And we're done ... licensed LGPL-3.0" footer cell. +# +# The boot-time notebook refresh uses this to avoid rewriting a user's notebook +# when only that boilerplate moved upstream. We hash ONLY the middle (the cells +# that are not install/announcement/footer) and compare. Outputs, execution +# counts, cell ids and metadata are ignored, so merely running a notebook never +# changes the signature. +# +# Usage: +# unsloth_nb_content_sig.py -> prints SAME | DIFF | ERR +# unsloth_nb_content_sig.py -> prints the middle digest +# +# Exit code is always 0; the decision is the printed word. On any parse problem +# we print ERR / nothing so the caller can fall back to its whole-file logic. +import hashlib +import json +import sys + +# Lowercased substrings that mark a markdown cell as top/bottom boilerplate. +_BOILERPLATE_MD = ( + "to run this, press", # Colab/AMD run announcement + 'press "*runtime*"', + "### news", # News heading + "introducing **unsloth studio**", # rotating announcement body + "you will learn how to do", # announcement tail + "this notebook is licensed", # announcement license line + "and we're done", # footer opener + "this notebook and all unsloth notebooks are licensed", # footer license + "join discord if you need help", # footer + "star us on", # footer + "some other resources", # footer resources block +) + + +def _text(cell): + src = cell.get("source", "") + if isinstance(src, list): + src = "".join(src) + return src.replace("\r\n", "\n").replace("\r", "\n") + + +def _is_install_code(cell): + if cell.get("cell_type") != "code": + return False + t = _text(cell) + low = t.lower() + if "pip install" in low or "pip3-autoremove" in low: + return True + first = t.lstrip().split("\n", 1)[0].strip().lower() + return first.startswith("%%capture") or first.startswith("%%bash") + + +def _is_boilerplate_md(cell): + if cell.get("cell_type") != "markdown": + return False + low = _text(cell).lower() + return any(m in low for m in _BOILERPLATE_MD) + + +def _is_boilerplate(cell): + return _is_install_code(cell) or _is_boilerplate_md(cell) + + +def middle_digest(path): + """sha256 over the (type, source) of every non-boilerplate cell, or None.""" + try: + with open(path, "r", encoding="utf-8") as f: + nb = json.load(f) + except Exception: + return None + cells = nb.get("cells") + if not isinstance(cells, list): + return None + h = hashlib.sha256() + for cell in cells: + if not isinstance(cell, dict): + continue + if _is_boilerplate(cell): + continue + h.update(b"\x00") + h.update(str(cell.get("cell_type", "")).encode("utf-8")) + h.update(b"\x01") + h.update(_text(cell).encode("utf-8")) + return h.hexdigest() + + +def main(argv): + if len(argv) == 2: + d = middle_digest(argv[1]) + if d is None: + print("ERR") + return 0 + print(d) + return 0 + if len(argv) == 3: + a = middle_digest(argv[1]) + b = middle_digest(argv[2]) + if a is None or b is None: + print("ERR") + elif a == b: + print("SAME") + else: + print("DIFF") + return 0 + print("ERR") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/docker/unsloth_sync_notebooks.sh b/docker/unsloth_sync_notebooks.sh index 7c50525ea3..cce87fb6f5 100644 --- a/docker/unsloth_sync_notebooks.sh +++ b/docker/unsloth_sync_notebooks.sh @@ -28,6 +28,33 @@ STATE="$DEST/.unsloth_sync_state" # "sha256 relpath" of what we last wrote SYNCED="$DEST/.unsloth_sync_commit" # upstream commit we last synced to TIMEOUT="${UNSLOTH_NOTEBOOK_FETCH_TIMEOUT:-60}" +# Helper that compares the *content* (the middle, ignoring the auto-generated +# install header / announcements / footer) of two notebooks. Used so a refresh +# doesn't rewrite an untouched notebook when only that boilerplate moved +# upstream. Resolved from an explicit override, then PATH, then a sibling file. +PYBIN="$(command -v python3 2>/dev/null || command -v python 2>/dev/null || true)" +SIG_HELPER="${UNSLOTH_NB_SIG_HELPER:-}" +if [ -z "$SIG_HELPER" ]; then + if command -v unsloth-nb-content-sig >/dev/null 2>&1; then + SIG_HELPER="$(command -v unsloth-nb-content-sig)" + else + _self_dir="$(cd "$(dirname "$0")" 2>/dev/null && pwd)" + [ -n "$_self_dir" ] && [ -f "$_self_dir/unsloth_nb_content_sig.py" ] \ + && SIG_HELPER="$_self_dir/unsloth_nb_content_sig.py" + fi +fi + +# True only when BOTH are .ipynb, the helper is usable, and it reports the +# non-boilerplate middle is identical (so only the header/footer changed). +# Any failure returns false, so the caller falls back to a normal refresh. +middle_unchanged() { + case "$1" in *.ipynb) : ;; *) return 1 ;; esac + [ -n "$PYBIN" ] && [ -n "$SIG_HELPER" ] || return 1 + [ "${UNSLOTH_NOTEBOOK_BODY_AWARE:-1}" = "1" ] || return 1 + [ "$("$PYBIN" "$SIG_HELPER" "$1" "$2" 2>/dev/null)" = "SAME" ] || return 1 + return 0 +} + [ "${UNSLOTH_SKIP_NOTEBOOK_SYNC:-0}" = "1" ] && exit 0 [ -d "$TEMPLATE" ] || exit 0 mkdir -p "$DEST" 2>/dev/null || exit 0 @@ -83,7 +110,7 @@ if [ -f "$STATE" ]; then fi TMPSTATE="$(mktemp)" -updated=0; kept=0 +updated=0; kept=0; unchanged=0 while IFS= read -r -d '' f; do rel="${f#"$TMP"/}" case "$rel" in .git|.git/*) continue ;; esac @@ -96,6 +123,14 @@ while IFS= read -r -d '' f; do kept=$((kept + 1)) continue fi + if [ -n "$rec" ] && middle_unchanged "$dst" "$f"; then + # Untouched notebook whose only upstream change is the install + # header / announcements / footer. The tutorial body is identical, + # so don't churn the user's file -- keep it and its marker as-is. + printf '%s %s\n' "$rec" "$rel" >> "$TMPSTATE" + unchanged=$((unchanged + 1)) + continue + fi fi mkdir -p "$(dirname "$dst")" 2>/dev/null || true if cp -a "$f" "$dst" 2>/dev/null; then @@ -107,5 +142,5 @@ done < <(find "$TMP" -type f -print0) mv "$TMPSTATE" "$STATE" 2>/dev/null || rm -f "$TMPSTATE" echo "$remote" > "$SYNCED" 2>/dev/null || true rm -rf "$TMP" -echo "[unsloth-nb] notebooks refreshed from GitHub: $updated updated, $kept kept (your edits)" +echo "[unsloth-nb] notebooks refreshed from GitHub: $updated updated, $kept kept (your edits), $unchanged kept (only header/footer changed upstream)" exit 0