fix(install): retry GitHub API 403 with Retry-After / X-RateLimit-Reset

Anonymous calls to api.github.com share a 60-req/hour bucket per
runner IP. CI fleets exhaust this trivially -- e.g. PR 5322 run
25490821956 / job 74798111390 hit 403 on the very first
ggml-org/llama.cpp /releases?per_page=100&page=1 call, fell back
to source build, and the workflow asserter then bailed because it
expects the prebuilt path to succeed. install_llama_prebuilt.py
gave up on 403 in one shot:

  raise RuntimeError(f"GitHub API returned 403 for {url}{hint}")

Now: treat 403 against api.github.com as retryable (real 403s on
other hosts -- private artefact downloads, auth failures -- stay
non-retryable). The existing download_bytes retry loop picks it
up automatically. sleep_backoff() takes an optional `exc=` and
honours the Retry-After / X-RateLimit-Reset headers so the wait
is accurate, capped at 60s (anything longer means the source
build fallback is faster than waiting). After all retries, the
existing RuntimeError surface is preserved -- callers fall back
to source build exactly as today, just less often.

Combined with passing GH_TOKEN to the install step (which the
Mac and Linux GGUF jobs on this branch already do, see e.g.
studio-inference-smoke.yml line 105), the prebuilt path is now
robust against both transient 403 blips AND sustained anonymous
rate-limit exhaustion: GH_TOKEN bumps the bucket from 60 to
5000 req/hour, and the new retry/header-honouring logic
absorbs the remaining flakes.
This commit is contained in:
Daniel Han 2026-05-07 11:23:30 +00:00
commit 9bfd4bc58c

View file

@ -430,6 +430,16 @@ def is_github_api_url(url: str | None) -> bool:
def is_retryable_url_error(exc: Exception) -> bool:
if isinstance(exc, urllib.error.HTTPError):
# GitHub returns 403 (not the standard 429) when the API rate
# limit is hit. Anonymous calls share a 60-req/hour bucket per
# runner IP, which CI fleets can exhaust trivially. Treat 403
# against api.github.com as retryable so we get one or two
# backoff cycles before the source-build fallback fires; honour
# Retry-After / X-RateLimit-Reset in sleep_backoff for accurate
# waits. Real 403s on other hosts (private artefact downloads,
# auth failures) stay non-retryable.
if exc.code == 403:
return is_github_api_url(getattr(exc, "url", None))
return exc.code in RETRYABLE_HTTP_STATUS
if isinstance(exc, urllib.error.URLError):
return True
@ -440,10 +450,43 @@ def is_retryable_url_error(exc: Exception) -> bool:
return False
_RATE_LIMIT_WAIT_CAP_SECONDS = 60.0
def _http_error_retry_delay(exc: Exception) -> float | None:
"""Extract a recommended wait from rate-limit headers on a 403/429.
Returns None when no header is present or the indicated wait is
longer than _RATE_LIMIT_WAIT_CAP_SECONDS (in which case the caller
should not block on it -- the source-build fallback is faster).
"""
if not isinstance(exc, urllib.error.HTTPError):
return None
headers = getattr(exc, "headers", None)
if headers is None:
return None
retry_after = headers.get("Retry-After")
if retry_after and retry_after.strip().isdigit():
wait = float(retry_after.strip())
return wait if wait <= _RATE_LIMIT_WAIT_CAP_SECONDS else None
rate_reset = headers.get("X-RateLimit-Reset")
if rate_reset and rate_reset.strip().isdigit():
wait = float(rate_reset.strip()) - time.time()
if 0.0 < wait <= _RATE_LIMIT_WAIT_CAP_SECONDS:
return wait + 1.0 # +1s of slack so the bucket is fresh
return None
def sleep_backoff(
attempt: int, *, base_delay: float = HTTP_FETCH_BASE_DELAY_SECONDS
attempt: int,
*,
base_delay: float = HTTP_FETCH_BASE_DELAY_SECONDS,
exc: Exception | None = None,
) -> None:
delay = base_delay * (2 ** max(attempt - 1, 0))
header_delay = _http_error_retry_delay(exc) if exc is not None else None
if header_delay is not None:
delay = max(delay, header_delay)
delay += random.uniform(0.0, 0.2)
time.sleep(delay)
@ -829,7 +872,7 @@ def download_bytes(
if attempt >= attempts or not is_retryable_url_error(exc):
raise
log(f"fetch failed ({attempt}/{attempts}) for {url}: {exc}; retrying")
sleep_backoff(attempt)
sleep_backoff(attempt, exc = exc)
assert last_exc is not None
raise last_exc
@ -927,7 +970,7 @@ def download_file(url: str, destination: Path) -> None:
log(
f"download failed ({attempt}/{HTTP_FETCH_ATTEMPTS}) for {url}: {exc}; retrying"
)
sleep_backoff(attempt)
sleep_backoff(attempt, exc = exc)
assert last_exc is not None
raise last_exc