CI: retry transient HTTP timeouts in Studio smoke probes (#7052)
* CI: retry transient HTTP timeouts in Studio smoke probes The post() helper in the Studio inference smoke workflows does a single urlopen with a 240s timeout against the local Studio server. On shared runners this sporadically hits TimeoutError while the server is stalled, failing the whole job for a transport hiccup; the same flake has recurred across unrelated PRs on Linux and Windows (JSON/images and tool-calling jobs) and passes on rerun. Retry the probe up to 3 times on transport-level failures only (TimeoutError, ConnectionError, non-HTTP URLError), 15s apart. HTTP status errors still surface immediately, so genuine server failures are unaffected. post_sse() is left unchanged: it has a 600s budget and has not flaked. * CI: retry only short probes so worst case fits the job budget Some json-images calls pass timeout=600; three attempts there could spend 30 minutes in one step and hit the job's timeout-minutes instead of failing with the Python error. Retry (3 attempts) only when timeout <= 300s, which covers the observed flaky 180-240s probes; longer probes keep the pre-PR single attempt. * CI: give long smoke probes one capped retry Round two of bounding the retries: timeout>300s probes previously got a single attempt, so a transient stall in the 600s JSON-mode probes still failed on first occurrence. Give them one retry with the attempt timeout capped at 300s. Worst cases stay inside timeout-minutes: 240s probes 12.5 min, one 600s probe 15.25 min, the Windows JSON job's two long probes 30.5 min against its 35 minute budget.
This commit is contained in:
Daniel Han
GitHub
parent
b0b8aea618
commit
fbcd3fa511
3 changed files with 120 additions and 12 deletions
44
.github/workflows/studio-inference-smoke.yml
vendored
44
.github/workflows/studio-inference-smoke.yml
vendored
|
|
@ -444,6 +444,8 @@ jobs:
|
|||
python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ["BASE_URL"]
|
||||
|
|
@ -464,8 +466,24 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately. Bounded
|
||||
# to fit the job's timeout-minutes: short probes get 3 full
|
||||
# attempts, long probes one retry capped at 300s (a healthy
|
||||
# server answers a retry quickly; a stalled one never does).
|
||||
attempts = 3 if timeout <= 300 else 2
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
t = timeout if attempt == 0 else min(timeout, 300)
|
||||
with urllib.request.urlopen(req, timeout = t) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
def post_sse(path, body, *, timeout = 600):
|
||||
"""POST a streaming request and accumulate the assistant
|
||||
|
|
@ -938,6 +956,8 @@ jobs:
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from openai import OpenAI
|
||||
from anthropic import Anthropic
|
||||
|
|
@ -956,8 +976,24 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately. Bounded
|
||||
# to fit the job's timeout-minutes: short probes get 3 full
|
||||
# attempts, long probes one retry capped at 300s (a healthy
|
||||
# server answers a retry quickly; a stalled one never does).
|
||||
attempts = 3 if timeout <= 300 else 2
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
t = timeout if attempt == 0 else min(timeout, 300)
|
||||
with urllib.request.urlopen(req, timeout = t) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
# ── 1. response_format = json_object (JSON mode) ─────────────
|
||||
# llama.cpp's HTTP server supports OpenAI-compatible JSON
|
||||
|
|
|
|||
44
.github/workflows/studio-mac-inference-smoke.yml
vendored
44
.github/workflows/studio-mac-inference-smoke.yml
vendored
|
|
@ -430,6 +430,8 @@ jobs:
|
|||
python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ["BASE_URL"]
|
||||
|
|
@ -450,8 +452,24 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately. Bounded
|
||||
# to fit the job's timeout-minutes: short probes get 3 full
|
||||
# attempts, long probes one retry capped at 300s (a healthy
|
||||
# server answers a retry quickly; a stalled one never does).
|
||||
attempts = 3 if timeout <= 300 else 2
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
t = timeout if attempt == 0 else min(timeout, 300)
|
||||
with urllib.request.urlopen(req, timeout = t) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
def post_sse(path, body, *, timeout = 600):
|
||||
"""POST a streaming request and accumulate the assistant
|
||||
|
|
@ -825,6 +843,8 @@ jobs:
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from openai import OpenAI
|
||||
from anthropic import Anthropic
|
||||
|
|
@ -848,8 +868,24 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately. Bounded
|
||||
# to fit the job's timeout-minutes: short probes get 3 full
|
||||
# attempts, long probes one retry capped at 300s (a healthy
|
||||
# server answers a retry quickly; a stalled one never does).
|
||||
attempts = 3 if timeout <= 300 else 2
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
t = timeout if attempt == 0 else min(timeout, 300)
|
||||
with urllib.request.urlopen(req, timeout = t) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
# ── 1. response_format = json_object (JSON mode) ─────────────
|
||||
# llama.cpp's HTTP server supports OpenAI-compatible JSON
|
||||
|
|
|
|||
|
|
@ -634,6 +634,8 @@ jobs:
|
|||
python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ["BASE_URL"]
|
||||
|
|
@ -656,8 +658,24 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately. Bounded
|
||||
# to fit the job's timeout-minutes: short probes get 3 full
|
||||
# attempts, long probes one retry capped at 300s (a healthy
|
||||
# server answers a retry quickly; a stalled one never does).
|
||||
attempts = 3 if timeout <= 300 else 2
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
t = timeout if attempt == 0 else min(timeout, 300)
|
||||
with urllib.request.urlopen(req, timeout = t) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
def post_sse(path, body, *, timeout = 600):
|
||||
body = {**body, "stream": True}
|
||||
|
|
@ -1063,6 +1081,8 @@ jobs:
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from openai import OpenAI
|
||||
from anthropic import Anthropic
|
||||
|
|
@ -1082,8 +1102,24 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately. Bounded
|
||||
# to fit the job's timeout-minutes: short probes get 3 full
|
||||
# attempts, long probes one retry capped at 300s (a healthy
|
||||
# server answers a retry quickly; a stalled one never does).
|
||||
attempts = 3 if timeout <= 300 else 2
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
t = timeout if attempt == 0 else min(timeout, 300)
|
||||
with urllib.request.urlopen(req, timeout = t) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
# ── 1. response_format = json_object (JSON mode) ─────────────
|
||||
status, data = post("/v1/chat/completions", {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue