CI: retry transient HTTP timeouts in Studio smoke probes
The post() helper in the Studio inference smoke workflows does a single urlopen with a 240s timeout against the local Studio server. On shared runners this sporadically hits TimeoutError while the server is stalled, failing the whole job for a transport hiccup; the same flake has recurred across unrelated PRs on Linux and Windows (JSON/images and tool-calling jobs) and passes on rerun. Retry the probe up to 3 times on transport-level failures only (TimeoutError, ConnectionError, non-HTTP URLError), 15s apart. HTTP status errors still surface immediately, so genuine server failures are unaffected. post_sse() is left unchanged: it has a 600s budget and has not flaked.
This commit is contained in:
Daniel Han
parent
b0b8aea618
commit
fc76ce3391
3 changed files with 90 additions and 12 deletions
34
.github/workflows/studio-inference-smoke.yml
vendored
34
.github/workflows/studio-inference-smoke.yml
vendored
|
|
@ -444,6 +444,8 @@ jobs:
|
|||
python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ["BASE_URL"]
|
||||
|
|
@ -464,8 +466,19 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
def post_sse(path, body, *, timeout = 600):
|
||||
"""POST a streaming request and accumulate the assistant
|
||||
|
|
@ -938,6 +951,8 @@ jobs:
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from openai import OpenAI
|
||||
from anthropic import Anthropic
|
||||
|
|
@ -956,8 +971,19 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
# ── 1. response_format = json_object (JSON mode) ─────────────
|
||||
# llama.cpp's HTTP server supports OpenAI-compatible JSON
|
||||
|
|
|
|||
34
.github/workflows/studio-mac-inference-smoke.yml
vendored
34
.github/workflows/studio-mac-inference-smoke.yml
vendored
|
|
@ -430,6 +430,8 @@ jobs:
|
|||
python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ["BASE_URL"]
|
||||
|
|
@ -450,8 +452,19 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
def post_sse(path, body, *, timeout = 600):
|
||||
"""POST a streaming request and accumulate the assistant
|
||||
|
|
@ -825,6 +838,8 @@ jobs:
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from openai import OpenAI
|
||||
from anthropic import Anthropic
|
||||
|
|
@ -848,8 +863,19 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
# ── 1. response_format = json_object (JSON mode) ─────────────
|
||||
# llama.cpp's HTTP server supports OpenAI-compatible JSON
|
||||
|
|
|
|||
|
|
@ -634,6 +634,8 @@ jobs:
|
|||
python - <<'PY'
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE = os.environ["BASE_URL"]
|
||||
|
|
@ -656,8 +658,19 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
def post_sse(path, body, *, timeout = 600):
|
||||
body = {**body, "stream": True}
|
||||
|
|
@ -1063,6 +1076,8 @@ jobs:
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from openai import OpenAI
|
||||
from anthropic import Anthropic
|
||||
|
|
@ -1082,8 +1097,19 @@ jobs:
|
|||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
# Shared CI runners stall sporadically, so retry transport-level
|
||||
# failures only; HTTP status errors surface immediately.
|
||||
for attempt in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
except urllib.error.HTTPError:
|
||||
raise
|
||||
except (TimeoutError, ConnectionError, urllib.error.URLError) as exc:
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f"[retry] {path}: {exc!r}", flush = True)
|
||||
time.sleep(15)
|
||||
|
||||
# ── 1. response_format = json_object (JSON mode) ─────────────
|
||||
status, data = post("/v1/chat/completions", {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue