From 54a86c351431834603d382e7a9f8f6c14c7f7049 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 15 May 2026 21:11:56 -0700 Subject: [PATCH] ci: route every `hf download` through xet-tuned stall-retry wrapper (#5476) Root cause of the Mac json-images 30 min timeout (run 25950714888 / PR #5430): huggingface_hub>=1.15 deprecated `hf_transfer` and routes every transfer through `hf-xet`. The CI step's unpinned `pip install --upgrade huggingface_hub hf_transfer` jumped to 1.15.0 + hf-xet 1.5.0, the 940 MB mmproj finished in ~21s, then the 3 GB gemma-4 GGUF made it to ~46% and went completely silent for the remaining 29 minutes -- no progress bytes, no error, no exit -- until the job timeout fired. This wraps every CI `hf download` in a new `.github/scripts/hf-download-with-retry.sh`: * Drops the no-op `HF_HUB_ENABLE_HF_TRANSFER=1` prefix and the `hf_transfer` install (both are deprecated on 1.15+ and only emit a FutureWarning now). * Exports the hf-xet high-performance knobs Daniel asked for: HF_XET_HIGH_PERFORMANCE=1 HF_XET_CHUNK_CACHE_SIZE_BYTES=0 HF_XET_NUM_CONCURRENT_RANGE_GETS=64 HF_XET_RECONSTRUCT_WRITE_SEQUENTIALLY=0 HF_XET_CLIENT_READ_TIMEOUT=500 * Watchdogs each attempt: if `hf download` has not exited after HF_DOWNLOAD_STALL_SECONDS (default 180s = 3 min), SIGTERM, sleep 2, SIGKILL, then loop. Retries are unbounded; the enclosing job's `timeout-minutes` is the real cap. * Optional 3rd positional `LOCAL_DIR` -- omitted lets `hf` use the default HF_HUB_CACHE, which is what the HF_HOME-priming jobs need. 19 call sites migrated across mlx-ci.yml + 9 studio-*-smoke.yml workflows. The inline `python -c "from huggingface_hub import hf_hub_download; ..."` block in mlx-ci.yml is also routed through the wrapper so every hf transfer in CI gets the same treatment. Also reverts the json-images timeout 45 -> 30 from #5475: the bump was masking this hang, not fixing it. --- .github/scripts/hf-download-with-retry.sh | 107 ++++++++++++++++++ .github/workflows/mlx-ci.yml | 13 +-- .github/workflows/studio-api-smoke.yml | 5 +- .github/workflows/studio-inference-smoke.yml | 18 ++- .github/workflows/studio-mac-api-smoke.yml | 5 +- .../workflows/studio-mac-inference-smoke.yml | 27 ++--- .github/workflows/studio-mac-ui-smoke.yml | 5 +- .github/workflows/studio-ui-smoke.yml | 5 +- .../workflows/studio-windows-api-smoke.yml | 5 +- .../studio-windows-inference-smoke.yml | 18 ++- .github/workflows/studio-windows-ui-smoke.yml | 5 +- 11 files changed, 145 insertions(+), 68 deletions(-) create mode 100755 .github/scripts/hf-download-with-retry.sh diff --git a/.github/scripts/hf-download-with-retry.sh b/.github/scripts/hf-download-with-retry.sh new file mode 100755 index 0000000000..c5ee013c80 --- /dev/null +++ b/.github/scripts/hf-download-with-retry.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# +# Download a single file from a Hugging Face repo with a stall-retry +# watchdog. Used by the Studio CI workflows so a hung hf-xet transfer +# kills + retries instead of silently consuming the job's timeout. +# +# Usage: hf-download-with-retry.sh REPO FILE LOCAL_DIR +# +# Why this exists +# --------------- +# huggingface_hub 1.15+ deprecated `hf_transfer` and routes every +# transfer through the `hf-xet` binary package. In CI we observed +# `hf download` on a 3 GB GGUF (gemma-4-E2B-it-UD-Q4_K_XL) progress +# to ~46% via Xet, then go completely silent for the remainder of +# the 30-min job timeout -- no progress bytes, no error, no exit. +# A sibling 940 MB mmproj on the same step downloaded in ~21s +# moments earlier, so the hang is per-file inside hf-xet rather +# than a network outage. The Xet env-vars below put hf-xet into +# its highest-throughput mode and force a 500 s client-read +# timeout; the watchdog loop ensures a stall does not eat the +# whole job: if the hf process has not exited after STALL_S +# seconds (default 180 = 3 min), we SIGTERM, then SIGKILL, then +# start a fresh attempt. Retries are unbounded -- the enclosing +# GitHub Actions job's `timeout-minutes` is the real bound. +# +# See https://huggingface.co/docs/huggingface_hub/package_reference/environment_variables +# for the HF_XET_* documentation, and npm/cli#7308's pattern (silent +# CI hang with no error) for prior art on this class of failure. + +set -uo pipefail + +REPO="${1:?usage: hf-download-with-retry.sh REPO FILE [LOCAL_DIR]}" +FILE="${2:?usage: hf-download-with-retry.sh REPO FILE [LOCAL_DIR]}" +# LOCAL_DIR is optional. If empty, hf falls back to HF_HUB_CACHE +# (~/.cache/huggingface/hub) which is the desired path for callers +# that populate HF_HOME for a downstream Studio model load. +LOCAL_DIR="${3:-}" + +# Stall threshold per attempt, in seconds. Override with +# HF_DOWNLOAD_STALL_SECONDS in the workflow env if 3 min is too tight +# for a specific runner / file. The script keeps retrying past this +# until the job timeout fires. +STALL_S="${HF_DOWNLOAD_STALL_SECONDS:-180}" + +# hf-xet tuning. HF_HUB_ENABLE_HF_TRANSFER is deliberately NOT set -- +# it is a no-op on huggingface_hub>=1.15 and only emits a deprecation +# FutureWarning. The five HF_XET_* knobs below mirror the settings +# Daniel asked for: max bandwidth + 64 parallel range gets, no chunk +# cache (download-once usage pattern), parallel disk writes (SSD/NVMe +# runners), and a generous 500 s read timeout so individual chunk +# requests fail loudly instead of stalling forever. +export HF_XET_HIGH_PERFORMANCE=1 +export HF_XET_CHUNK_CACHE_SIZE_BYTES=0 +export HF_XET_NUM_CONCURRENT_RANGE_GETS=64 +export HF_XET_RECONSTRUCT_WRITE_SEQUENTIALLY=0 +export HF_XET_CLIENT_READ_TIMEOUT=500 + +if [ -n "$LOCAL_DIR" ]; then + mkdir -p "$LOCAL_DIR" +fi + +attempt=1 +while : ; do + log="$(mktemp -t hf-download.XXXXXX)" + echo "[hf-download] $FILE attempt $attempt (stall threshold ${STALL_S}s, log=$log)" + + if [ -n "$LOCAL_DIR" ]; then + hf download "$REPO" "$FILE" --local-dir "$LOCAL_DIR" > "$log" 2>&1 & + else + hf download "$REPO" "$FILE" > "$log" 2>&1 & + fi + pid=$! + + elapsed=0 + while kill -0 "$pid" 2>/dev/null && [ "$elapsed" -lt "$STALL_S" ]; do + sleep 5 + elapsed=$((elapsed + 5)) + done + + if kill -0 "$pid" 2>/dev/null; then + echo "[hf-download] $FILE attempt $attempt exceeded ${STALL_S}s -- killing PID $pid and retrying" + kill -TERM "$pid" 2>/dev/null || true + sleep 2 + kill -KILL "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + echo "[hf-download] $FILE attempt $attempt log tail (last 40 lines):" + tail -40 "$log" || true + attempt=$((attempt + 1)) + continue + fi + + if wait "$pid"; then + rc=0 + else + rc=$? + fi + + if [ "$rc" -eq 0 ]; then + echo "[hf-download] $FILE attempt $attempt succeeded" + tail -20 "$log" || true + exit 0 + fi + + echo "[hf-download] $FILE attempt $attempt failed (exit $rc) -- retrying" + tail -40 "$log" || true + attempt=$((attempt + 1)) +done diff --git a/.github/workflows/mlx-ci.yml b/.github/workflows/mlx-ci.yml index 8cd95bd30a..75940832a0 100644 --- a/.github/workflows/mlx-ci.yml +++ b/.github/workflows/mlx-ci.yml @@ -302,15 +302,10 @@ jobs: "$LLAMA_QUANT" --help >/dev/null && echo " llama-quantize loads OK" mkdir -p /tmp/ggufs - python -c " - from huggingface_hub import hf_hub_download - p = hf_hub_download( - 'unsloth/gemma-3-270m-it-GGUF', - 'gemma-3-270m-it-Q4_K_M.gguf', - local_dir = '/tmp/ggufs', - ) - print('downloaded:', p) - " + bash .github/scripts/hf-download-with-retry.sh \ + 'unsloth/gemma-3-270m-it-GGUF' \ + 'gemma-3-270m-it-Q4_K_M.gguf' \ + /tmp/ggufs PORT=18080 echo "=== starting llama-server on 127.0.0.1:$PORT ===" diff --git a/.github/workflows/studio-api-smoke.yml b/.github/workflows/studio-api-smoke.yml index 8bc2acc3b0..53514e2ce1 100644 --- a/.github/workflows/studio-api-smoke.yml +++ b/.github/workflows/studio-api-smoke.yml @@ -85,10 +85,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} if: always() && steps.prime-hf.outcome == 'success' diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index 0f2320824c..775363e73c 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -99,10 +99,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} if: always() && steps.prime-hf.outcome == 'success' @@ -347,10 +346,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p gguf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache - name: Save GGUF model file if: always() && steps.download-gguf.outcome == 'success' @@ -664,12 +662,10 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$MMPROJ_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$MMPROJ_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} (model + mmproj) if: always() && steps.prime-hf.outcome == 'success' diff --git a/.github/workflows/studio-mac-api-smoke.yml b/.github/workflows/studio-mac-api-smoke.yml index 9e57b097ab..b4e274155e 100644 --- a/.github/workflows/studio-mac-api-smoke.yml +++ b/.github/workflows/studio-mac-api-smoke.yml @@ -70,10 +70,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} if: always() && steps.prime-hf.outcome == 'success' diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index b1efa238f3..2d6864e0cb 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -93,10 +93,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" # Save partial caches on cancel/timeout -- hf download resumes by # content hash. `outcome != skipped` keeps cache-hit a no-op. @@ -343,10 +342,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p gguf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache # Save partial caches on cancel; next run resumes via content hash. - name: Save GGUF model file @@ -668,14 +666,7 @@ jobs: json-images: name: JSON, images runs-on: macos-14 - # 45 min, not 30. The job downloads ~4 GB on a cache miss - # (3 GB gemma-4 GGUF + ~1 GB mmproj) over shared macos-14 NAT, - # plus Studio install + boot + JSON/image smoke. The previous - # 30 min cap timed out cache-miss runs mid-download (run - # 25950714888 / PR #5430). Once the cache is warm it lands in - # ~10 min; the headroom only matters on the first run after a - # cache key bump (v1->v2 in #5459). - timeout-minutes: 45 + timeout-minutes: 30 env: GGUF_REPO: unsloth/gemma-4-E2B-it-GGUF # Linux smoke uses UD-IQ3_XXS, but on Mac Metal that gemma-4 @@ -732,13 +723,11 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p gguf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache & + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache & MODEL_PID=$! - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$MMPROJ_FILE" --local-dir gguf-cache & + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$MMPROJ_FILE" gguf-cache & MMPROJ_PID=$! wait "$MODEL_PID" wait "$MMPROJ_PID" diff --git a/.github/workflows/studio-mac-ui-smoke.yml b/.github/workflows/studio-mac-ui-smoke.yml index c9742c543a..510c3543d2 100644 --- a/.github/workflows/studio-mac-ui-smoke.yml +++ b/.github/workflows/studio-mac-ui-smoke.yml @@ -70,10 +70,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} if: always() && steps.prime-hf.outcome == 'success' diff --git a/.github/workflows/studio-ui-smoke.yml b/.github/workflows/studio-ui-smoke.yml index cd0765a1e3..79476a62ea 100644 --- a/.github/workflows/studio-ui-smoke.yml +++ b/.github/workflows/studio-ui-smoke.yml @@ -84,10 +84,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} if: always() && steps.prime-hf.outcome == 'success' diff --git a/.github/workflows/studio-windows-api-smoke.yml b/.github/workflows/studio-windows-api-smoke.yml index d5eaa75c35..1d12ea6f90 100644 --- a/.github/workflows/studio-windows-api-smoke.yml +++ b/.github/workflows/studio-windows-api-smoke.yml @@ -77,10 +77,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} if: always() && steps.prime-hf.outcome == 'success' diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index 146b49d4f6..01bf4127a7 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -99,10 +99,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME cache for ${{ env.GGUF_REPO }} # Only write a fresh cache entry when we actually rebuilt the @@ -420,10 +419,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p gguf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" --local-dir gguf-cache + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" gguf-cache - name: Save GGUF model cache if: always() && steps.download-gguf.outcome == 'success' @@ -835,12 +833,10 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$MMPROJ_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$MMPROJ_FILE" - name: Save HF_HOME cache for ${{ env.GGUF_REPO }} (model + mmproj) if: always() && steps.prime-hf.outcome == 'success' diff --git a/.github/workflows/studio-windows-ui-smoke.yml b/.github/workflows/studio-windows-ui-smoke.yml index da8bff80b7..e5ab9f8ab7 100644 --- a/.github/workflows/studio-windows-ui-smoke.yml +++ b/.github/workflows/studio-windows-ui-smoke.yml @@ -93,10 +93,9 @@ jobs: env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: | - python -m pip install --upgrade huggingface_hub hf_transfer + python -m pip install --upgrade huggingface_hub mkdir -p hf-cache - HF_HUB_ENABLE_HF_TRANSFER=1 \ - hf download "$GGUF_REPO" "$GGUF_FILE" + bash .github/scripts/hf-download-with-retry.sh "$GGUF_REPO" "$GGUF_FILE" - name: Save HF_HOME for ${{ env.GGUF_REPO }} if: always() && steps.prime-hf.outcome == 'success'