Tighten retry comments

CI: give long smoke probes one capped retry
Round two of bounding the retries: timeout>300s probes previously got a single attempt, so a transient stall in the 600s JSON-mode probes still failed on first occurrence. Give them one retry with the attempt timeout capped at 300s. Worst cases stay inside timeout-minutes: 240s probes 12.5 min, one 600s probe 15.25 min, the Windows JSON job's two long probes 30.5 min against its 35 minute budget.
2026-07-10 11:42:24 +00:00 · 2026-07-10 08:50:05 +00:00 · 2026-07-10 08:38:31 +00:00 · 2026-07-10 05:11:57 +00:00
3 changed files with 102 additions and 12 deletions
--- a/.github/workflows/studio-inference-smoke.yml
+++ b/.github/workflows/studio-inference-smoke.yml
@ -444,6 +444,8 @@ jobs:
          python - <<'PY'
          import json
          import os
+          import time
+          import urllib.error
          import urllib.request

          BASE = os.environ["BASE_URL"]
@ -464,8 +466,21 @@ jobs:
                      "Content-Type": "application/json",
                  },
              )
-              with urllib.request.urlopen(req, timeout = timeout) as resp:
-                  return resp.status, json.loads(resp.read().decode())
+              # Retry transport failures only (HTTP errors raise immediately);
+              # short probes get 3 attempts, long probes 1 retry capped at 300s.
+              attempts = 3 if timeout <= 300 else 2
+              for attempt in range(attempts):
+                  try:
+                      t = timeout if attempt == 0 else min(timeout, 300)
+                      with urllib.request.urlopen(req, timeout = t) as resp:
+                          return resp.status, json.loads(resp.read().decode())
+                  except urllib.error.HTTPError:
+                      raise
+                  except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
+                      if attempt == attempts - 1:
+                          raise
+                      print(f"[retry] {path}: {exc!r}", flush = True)
+                      time.sleep(15)

          def post_sse(path, body, *, timeout = 600):
              """POST a streaming request and accumulate the assistant
@ -938,6 +953,8 @@ jobs:
          import base64
          import json
          import os
+          import time
+          import urllib.error
          import urllib.request
          from openai import OpenAI
          from anthropic import Anthropic
@ -956,8 +973,21 @@ jobs:
                      "Content-Type":  "application/json",
                  },
              )
-              with urllib.request.urlopen(req, timeout = timeout) as resp:
-                  return resp.status, json.loads(resp.read().decode())
+              # Retry transport failures only (HTTP errors raise immediately);
+              # short probes get 3 attempts, long probes 1 retry capped at 300s.
+              attempts = 3 if timeout <= 300 else 2
+              for attempt in range(attempts):
+                  try:
+                      t = timeout if attempt == 0 else min(timeout, 300)
+                      with urllib.request.urlopen(req, timeout = t) as resp:
+                          return resp.status, json.loads(resp.read().decode())
+                  except urllib.error.HTTPError:
+                      raise
+                  except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
+                      if attempt == attempts - 1:
+                          raise
+                      print(f"[retry] {path}: {exc!r}", flush = True)
+                      time.sleep(15)

          # ── 1. response_format = json_object (JSON mode) ─────────────
          # llama.cpp's HTTP server supports OpenAI-compatible JSON
--- a/.github/workflows/studio-mac-inference-smoke.yml
+++ b/.github/workflows/studio-mac-inference-smoke.yml
@ -430,6 +430,8 @@ jobs:
          python - <<'PY'
          import json
          import os
+          import time
+          import urllib.error
          import urllib.request

          BASE = os.environ["BASE_URL"]
@ -450,8 +452,21 @@ jobs:
                      "Content-Type": "application/json",
                  },
              )
-              with urllib.request.urlopen(req, timeout = timeout) as resp:
-                  return resp.status, json.loads(resp.read().decode())
+              # Retry transport failures only (HTTP errors raise immediately);
+              # short probes get 3 attempts, long probes 1 retry capped at 300s.
+              attempts = 3 if timeout <= 300 else 2
+              for attempt in range(attempts):
+                  try:
+                      t = timeout if attempt == 0 else min(timeout, 300)
+                      with urllib.request.urlopen(req, timeout = t) as resp:
+                          return resp.status, json.loads(resp.read().decode())
+                  except urllib.error.HTTPError:
+                      raise
+                  except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
+                      if attempt == attempts - 1:
+                          raise
+                      print(f"[retry] {path}: {exc!r}", flush = True)
+                      time.sleep(15)

          def post_sse(path, body, *, timeout = 600):
              """POST a streaming request and accumulate the assistant
@ -825,6 +840,8 @@ jobs:
          import base64
          import json
          import os
+          import time
+          import urllib.error
          import urllib.request
          from openai import OpenAI
          from anthropic import Anthropic
@ -848,8 +865,21 @@ jobs:
                      "Content-Type":  "application/json",
                  },
              )
-              with urllib.request.urlopen(req, timeout = timeout) as resp:
-                  return resp.status, json.loads(resp.read().decode())
+              # Retry transport failures only (HTTP errors raise immediately);
+              # short probes get 3 attempts, long probes 1 retry capped at 300s.
+              attempts = 3 if timeout <= 300 else 2
+              for attempt in range(attempts):
+                  try:
+                      t = timeout if attempt == 0 else min(timeout, 300)
+                      with urllib.request.urlopen(req, timeout = t) as resp:
+                          return resp.status, json.loads(resp.read().decode())
+                  except urllib.error.HTTPError:
+                      raise
+                  except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
+                      if attempt == attempts - 1:
+                          raise
+                      print(f"[retry] {path}: {exc!r}", flush = True)
+                      time.sleep(15)

          # ── 1. response_format = json_object (JSON mode) ─────────────
          # llama.cpp's HTTP server supports OpenAI-compatible JSON
--- a/.github/workflows/studio-windows-inference-smoke.yml
+++ b/.github/workflows/studio-windows-inference-smoke.yml
@ -634,6 +634,8 @@ jobs:
          python - <<'PY'
          import json
          import os
+          import time
+          import urllib.error
          import urllib.request

          BASE = os.environ["BASE_URL"]
@ -656,8 +658,21 @@ jobs:
                      "Content-Type": "application/json",
                  },
              )
-              with urllib.request.urlopen(req, timeout = timeout) as resp:
-                  return resp.status, json.loads(resp.read().decode())
+              # Retry transport failures only (HTTP errors raise immediately);
+              # short probes get 3 attempts, long probes 1 retry capped at 300s.
+              attempts = 3 if timeout <= 300 else 2
+              for attempt in range(attempts):
+                  try:
+                      t = timeout if attempt == 0 else min(timeout, 300)
+                      with urllib.request.urlopen(req, timeout = t) as resp:
+                          return resp.status, json.loads(resp.read().decode())
+                  except urllib.error.HTTPError:
+                      raise
+                  except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
+                      if attempt == attempts - 1:
+                          raise
+                      print(f"[retry] {path}: {exc!r}", flush = True)
+                      time.sleep(15)

          def post_sse(path, body, *, timeout = 600):
              body = {**body, "stream": True}
@ -1063,6 +1078,8 @@ jobs:
          import base64
          import json
          import os
+          import time
+          import urllib.error
          import urllib.request
          from openai import OpenAI
          from anthropic import Anthropic
@ -1082,8 +1099,21 @@ jobs:
                      "Content-Type":  "application/json",
                  },
              )
-              with urllib.request.urlopen(req, timeout = timeout) as resp:
-                  return resp.status, json.loads(resp.read().decode())
+              # Retry transport failures only (HTTP errors raise immediately);
+              # short probes get 3 attempts, long probes 1 retry capped at 300s.
+              attempts = 3 if timeout <= 300 else 2
+              for attempt in range(attempts):
+                  try:
+                      t = timeout if attempt == 0 else min(timeout, 300)
+                      with urllib.request.urlopen(req, timeout = t) as resp:
+                          return resp.status, json.loads(resp.read().decode())
+                  except urllib.error.HTTPError:
+                      raise
+                  except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
+                      if attempt == attempts - 1:
+                          raise
+                      print(f"[retry] {path}: {exc!r}", flush = True)
+                      time.sleep(15)

          # ── 1. response_format = json_object (JSON mode) ─────────────
          status, data = post("/v1/chat/completions", {
Author	SHA1	Message	Date
danielhanchen	33da775312	Tighten retry comments	2026-07-10 11:42:24 +00:00
Daniel Han	e739dc1cca	CI: give long smoke probes one capped retry Round two of bounding the retries: timeout>300s probes previously got a single attempt, so a transient stall in the 600s JSON-mode probes still failed on first occurrence. Give them one retry with the attempt timeout capped at 300s. Worst cases stay inside timeout-minutes: 240s probes 12.5 min, one 600s probe 15.25 min, the Windows JSON job's two long probes 30.5 min against its 35 minute budget.	2026-07-10 08:50:05 +00:00
Daniel Han	b4c02db66f	CI: retry only short probes so worst case fits the job budget Some json-images calls pass timeout=600; three attempts there could spend 30 minutes in one step and hit the job's timeout-minutes instead of failing with the Python error. Retry (3 attempts) only when timeout <= 300s, which covers the observed flaky 180-240s probes; longer probes keep the pre-PR single attempt.	2026-07-10 08:38:31 +00:00
Daniel Han	fc76ce3391	CI: retry transient HTTP timeouts in Studio smoke probes The post() helper in the Studio inference smoke workflows does a single urlopen with a 240s timeout against the local Studio server. On shared runners this sporadically hits TimeoutError while the server is stalled, failing the whole job for a transport hiccup; the same flake has recurred across unrelated PRs on Linux and Windows (JSON/images and tool-calling jobs) and passes on rerun. Retry the probe up to 3 times on transport-level failures only (TimeoutError, ConnectionError, non-HTTP URLError), 15s apart. HTTP status errors still surface immediately, so genuine server failures are unaffected. post_sse() is left unchanged: it has a 600s budget and has not flaked.	2026-07-10 05:11:57 +00:00