From e739dc1cca312e3041aac4dd60a7928f11e00026 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 10 Jul 2026 08:50:05 +0000 Subject: [PATCH] CI: give long smoke probes one capped retry Round two of bounding the retries: timeout>300s probes previously got a single attempt, so a transient stall in the 600s JSON-mode probes still failed on first occurrence. Give them one retry with the attempt timeout capped at 300s. Worst cases stay inside timeout-minutes: 240s probes 12.5 min, one 600s probe 15.25 min, the Windows JSON job's two long probes 30.5 min against its 35 minute budget. --- .github/workflows/studio-inference-smoke.yml | 24 +++++++++++-------- .../workflows/studio-mac-inference-smoke.yml | 24 +++++++++++-------- .../studio-windows-inference-smoke.yml | 24 +++++++++++-------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index ce4bf46425..f540c11da4 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -467,13 +467,15 @@ jobs: }, ) # Shared CI runners stall sporadically, so retry transport-level - # failures only; HTTP status errors surface immediately. Long - # probes (timeout > 300s) get a single attempt so the worst - # case stays inside the job's timeout-minutes budget. - attempts = 3 if timeout <= 300 else 1 + # failures only; HTTP status errors surface immediately. Bounded + # to fit the job's timeout-minutes: short probes get 3 full + # attempts, long probes one retry capped at 300s (a healthy + # server answers a retry quickly; a stalled one never does). + attempts = 3 if timeout <= 300 else 2 for attempt in range(attempts): try: - with urllib.request.urlopen(req, timeout = timeout) as resp: + t = timeout if attempt == 0 else min(timeout, 300) + with urllib.request.urlopen(req, timeout = t) as resp: return resp.status, json.loads(resp.read().decode()) except urllib.error.HTTPError: raise @@ -975,13 +977,15 @@ jobs: }, ) # Shared CI runners stall sporadically, so retry transport-level - # failures only; HTTP status errors surface immediately. Long - # probes (timeout > 300s) get a single attempt so the worst - # case stays inside the job's timeout-minutes budget. - attempts = 3 if timeout <= 300 else 1 + # failures only; HTTP status errors surface immediately. Bounded + # to fit the job's timeout-minutes: short probes get 3 full + # attempts, long probes one retry capped at 300s (a healthy + # server answers a retry quickly; a stalled one never does). + attempts = 3 if timeout <= 300 else 2 for attempt in range(attempts): try: - with urllib.request.urlopen(req, timeout = timeout) as resp: + t = timeout if attempt == 0 else min(timeout, 300) + with urllib.request.urlopen(req, timeout = t) as resp: return resp.status, json.loads(resp.read().decode()) except urllib.error.HTTPError: raise diff --git a/.github/workflows/studio-mac-inference-smoke.yml b/.github/workflows/studio-mac-inference-smoke.yml index 805817b518..03c0a8580d 100644 --- a/.github/workflows/studio-mac-inference-smoke.yml +++ b/.github/workflows/studio-mac-inference-smoke.yml @@ -453,13 +453,15 @@ jobs: }, ) # Shared CI runners stall sporadically, so retry transport-level - # failures only; HTTP status errors surface immediately. Long - # probes (timeout > 300s) get a single attempt so the worst - # case stays inside the job's timeout-minutes budget. - attempts = 3 if timeout <= 300 else 1 + # failures only; HTTP status errors surface immediately. Bounded + # to fit the job's timeout-minutes: short probes get 3 full + # attempts, long probes one retry capped at 300s (a healthy + # server answers a retry quickly; a stalled one never does). + attempts = 3 if timeout <= 300 else 2 for attempt in range(attempts): try: - with urllib.request.urlopen(req, timeout = timeout) as resp: + t = timeout if attempt == 0 else min(timeout, 300) + with urllib.request.urlopen(req, timeout = t) as resp: return resp.status, json.loads(resp.read().decode()) except urllib.error.HTTPError: raise @@ -867,13 +869,15 @@ jobs: }, ) # Shared CI runners stall sporadically, so retry transport-level - # failures only; HTTP status errors surface immediately. Long - # probes (timeout > 300s) get a single attempt so the worst - # case stays inside the job's timeout-minutes budget. - attempts = 3 if timeout <= 300 else 1 + # failures only; HTTP status errors surface immediately. Bounded + # to fit the job's timeout-minutes: short probes get 3 full + # attempts, long probes one retry capped at 300s (a healthy + # server answers a retry quickly; a stalled one never does). + attempts = 3 if timeout <= 300 else 2 for attempt in range(attempts): try: - with urllib.request.urlopen(req, timeout = timeout) as resp: + t = timeout if attempt == 0 else min(timeout, 300) + with urllib.request.urlopen(req, timeout = t) as resp: return resp.status, json.loads(resp.read().decode()) except urllib.error.HTTPError: raise diff --git a/.github/workflows/studio-windows-inference-smoke.yml b/.github/workflows/studio-windows-inference-smoke.yml index 740cde15fb..0453c9212a 100644 --- a/.github/workflows/studio-windows-inference-smoke.yml +++ b/.github/workflows/studio-windows-inference-smoke.yml @@ -659,13 +659,15 @@ jobs: }, ) # Shared CI runners stall sporadically, so retry transport-level - # failures only; HTTP status errors surface immediately. Long - # probes (timeout > 300s) get a single attempt so the worst - # case stays inside the job's timeout-minutes budget. - attempts = 3 if timeout <= 300 else 1 + # failures only; HTTP status errors surface immediately. Bounded + # to fit the job's timeout-minutes: short probes get 3 full + # attempts, long probes one retry capped at 300s (a healthy + # server answers a retry quickly; a stalled one never does). + attempts = 3 if timeout <= 300 else 2 for attempt in range(attempts): try: - with urllib.request.urlopen(req, timeout = timeout) as resp: + t = timeout if attempt == 0 else min(timeout, 300) + with urllib.request.urlopen(req, timeout = t) as resp: return resp.status, json.loads(resp.read().decode()) except urllib.error.HTTPError: raise @@ -1101,13 +1103,15 @@ jobs: }, ) # Shared CI runners stall sporadically, so retry transport-level - # failures only; HTTP status errors surface immediately. Long - # probes (timeout > 300s) get a single attempt so the worst - # case stays inside the job's timeout-minutes budget. - attempts = 3 if timeout <= 300 else 1 + # failures only; HTTP status errors surface immediately. Bounded + # to fit the job's timeout-minutes: short probes get 3 full + # attempts, long probes one retry capped at 300s (a healthy + # server answers a retry quickly; a stalled one never does). + attempts = 3 if timeout <= 300 else 2 for attempt in range(attempts): try: - with urllib.request.urlopen(req, timeout = timeout) as resp: + t = timeout if attempt == 0 else min(timeout, 300) + with urllib.request.urlopen(req, timeout = t) as resp: return resp.status, json.loads(resp.read().decode()) except urllib.error.HTTPError: raise