ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

CI: retry only short probes so worst case fits the job budget

Browse source 
Some json-images calls pass timeout=600; three attempts there could spend
30 minutes in one step and hit the job's timeout-minutes instead of failing
with the Python error. Retry (3 attempts) only when timeout <= 300s, which
covers the observed flaky 180-240s probes; longer probes keep the pre-PR
single attempt.
This commit is contained in:
Daniel Han 2026-07-10 08:38:31 +00:00
commit b4c02db66f
3 changed files with 36 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

18
.github/workflows/studio-inference-smoke.yml vendored
View file

@ -467,15 +467,18 @@ jobs:
},
)
# Shared CI runners stall sporadically, so retry transport-level
# failures only; HTTP status errors surface immediately.
for attempt in range(3):
# failures only; HTTP status errors surface immediately. Long
# probes (timeout > 300s) get a single attempt so the worst
# case stays inside the job's timeout-minutes budget.
attempts = 3 if timeout <= 300 else 1
for attempt in range(attempts):
try:
with urllib.request.urlopen(req, timeout = timeout) as resp:
return resp.status, json.loads(resp.read().decode())
except urllib.error.HTTPError:
raise
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
if attempt == 2:
if attempt == attempts - 1:
raise
print(f"[retry] {path}: {exc!r}", flush = True)
time.sleep(15)
@ -972,15 +975,18 @@ jobs:
},
)
# Shared CI runners stall sporadically, so retry transport-level
# failures only; HTTP status errors surface immediately.
for attempt in range(3):
# failures only; HTTP status errors surface immediately. Long
# probes (timeout > 300s) get a single attempt so the worst
# case stays inside the job's timeout-minutes budget.
attempts = 3 if timeout <= 300 else 1
for attempt in range(attempts):
try:
with urllib.request.urlopen(req, timeout = timeout) as resp:
return resp.status, json.loads(resp.read().decode())
except urllib.error.HTTPError:
raise
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
if attempt == 2:
if attempt == attempts - 1:
raise
print(f"[retry] {path}: {exc!r}", flush = True)
time.sleep(15)

18
.github/workflows/studio-mac-inference-smoke.yml vendored
View file

@ -453,15 +453,18 @@ jobs:
},
)
# Shared CI runners stall sporadically, so retry transport-level
# failures only; HTTP status errors surface immediately.
for attempt in range(3):
# failures only; HTTP status errors surface immediately. Long
# probes (timeout > 300s) get a single attempt so the worst
# case stays inside the job's timeout-minutes budget.
attempts = 3 if timeout <= 300 else 1
for attempt in range(attempts):
try:
with urllib.request.urlopen(req, timeout = timeout) as resp:
return resp.status, json.loads(resp.read().decode())
except urllib.error.HTTPError:
raise
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
if attempt == 2:
if attempt == attempts - 1:
raise
print(f"[retry] {path}: {exc!r}", flush = True)
time.sleep(15)
@ -864,15 +867,18 @@ jobs:
},
)
# Shared CI runners stall sporadically, so retry transport-level
# failures only; HTTP status errors surface immediately.
for attempt in range(3):
# failures only; HTTP status errors surface immediately. Long
# probes (timeout > 300s) get a single attempt so the worst
# case stays inside the job's timeout-minutes budget.
attempts = 3 if timeout <= 300 else 1
for attempt in range(attempts):
try:
with urllib.request.urlopen(req, timeout = timeout) as resp:
return resp.status, json.loads(resp.read().decode())
except urllib.error.HTTPError:
raise
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
if attempt == 2:
if attempt == attempts - 1:
raise
print(f"[retry] {path}: {exc!r}", flush = True)
time.sleep(15)

18
.github/workflows/studio-windows-inference-smoke.yml vendored
View file

@ -659,15 +659,18 @@ jobs:
},
)
# Shared CI runners stall sporadically, so retry transport-level
# failures only; HTTP status errors surface immediately.
for attempt in range(3):
# failures only; HTTP status errors surface immediately. Long
# probes (timeout > 300s) get a single attempt so the worst
# case stays inside the job's timeout-minutes budget.
attempts = 3 if timeout <= 300 else 1
for attempt in range(attempts):
try:
with urllib.request.urlopen(req, timeout = timeout) as resp:
return resp.status, json.loads(resp.read().decode())
except urllib.error.HTTPError:
raise
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
if attempt == 2:
if attempt == attempts - 1:
raise
print(f"[retry] {path}: {exc!r}", flush = True)
time.sleep(15)
@ -1098,15 +1101,18 @@ jobs:
},
)
# Shared CI runners stall sporadically, so retry transport-level
# failures only; HTTP status errors surface immediately.
for attempt in range(3):
# failures only; HTTP status errors surface immediately. Long
# probes (timeout > 300s) get a single attempt so the worst
# case stays inside the job's timeout-minutes budget.
attempts = 3 if timeout <= 300 else 1
for attempt in range(attempts):
try:
with urllib.request.urlopen(req, timeout = timeout) as resp:
return resp.status, json.loads(resp.read().decode())
except urllib.error.HTTPError:
raise
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
if attempt == 2:
if attempt == attempts - 1:
raise
print(f"[retry] {path}: {exc!r}", flush = True)
time.sleep(15)