CI: fix three regressions in the new Studio GGUF jobs
Job 1 (OpenAI, Anthropic API tests):
Anthropic SDK appends /v1/messages to base_url itself, so passing
base_url=f"{BASE}/v1" produced /v1/v1/messages and 405'd. Bare BASE
is correct (matches the docs' "the SDK appends /v1 automatically").
OpenAI SDK side already worked: 4-turn transcript was fully
deterministic across two runs and the "Paris" sanity assertion
passed.
Job 2 (tool calling tests):
Booting with --enable-tools forces the process-level tool policy to
True for every request (state/tool_policy.py:get_tool_policy), which
hijacked the "Standard OpenAI function calling" test through the
server-side agentic loop -- the model called web_search instead of
returning structured tool_calls for the user's `weather_tool`. Drop
--enable-tools so policy is None (per-request honour). The python /
terminal / web_search probes already pass enable_tools=True
explicitly in their request bodies, so they keep working.
Job 3 (JSON, images):
Two issues. (a) The OpenAI Python SDK rewrites
response_format={"type":"json_schema",...} into something Studio's
llama-server backend doesn't accept, so resp came back as the raw
error string and resp.choices[0] tripped 'str has no attribute
choices'. Switched to raw HTTP with the `{"type":"json_object",
"schema":...}` form llama-server actually supports
(GBNF-from-schema, llama-server extension). (b) Anthropic SDK
base_url same fix as job 1.
This commit is contained in:
parent
8e19065be5
commit
bb1f4ed5dc
1 changed files with 50 additions and 25 deletions
75
.github/workflows/studio-inference-smoke.yml
vendored
75
.github/workflows/studio-inference-smoke.yml
vendored
|
|
@ -213,7 +213,10 @@ jobs:
|
|||
return replies
|
||||
|
||||
def run_anthropic():
|
||||
client = Anthropic(base_url = f"{BASE}/v1", api_key = KEY)
|
||||
# Anthropic SDK appends /v1/messages itself, so base_url
|
||||
# must NOT include /v1 (otherwise the request hits
|
||||
# /v1/v1/messages and 405s).
|
||||
client = Anthropic(base_url = BASE, api_key = KEY)
|
||||
history, replies = [], []
|
||||
for prompt in PROMPTS:
|
||||
history.append({"role": "user", "content": prompt})
|
||||
|
|
@ -325,13 +328,16 @@ jobs:
|
|||
mkdir -p logs
|
||||
# `unsloth studio run` boots the server, loads the GGUF via
|
||||
# the HF_HOME-cached path, and prints the API key on the
|
||||
# banner. --enable-tools makes the server-side tool registry
|
||||
# (python / terminal / web_search) available behind the
|
||||
# enable_tools=true request flag.
|
||||
# banner. We deliberately do NOT pass --enable-tools: that
|
||||
# forces the process-level tool policy to True, which
|
||||
# hijacks every request through the server-side agentic
|
||||
# loop and breaks the function-calling test below. Default
|
||||
# policy (None) honours each request's `enable_tools` flag,
|
||||
# which is what the script under "Tool calling" relies on.
|
||||
unsloth studio run \
|
||||
--model "$GGUF_REPO" --gguf-variant "$GGUF_VARIANT" \
|
||||
--port "$STUDIO_PORT" --host 127.0.0.1 \
|
||||
--enable-tools -y \
|
||||
-y \
|
||||
> logs/studio.log 2>&1 &
|
||||
echo "STUDIO_PID=$!" >> "$GITHUB_ENV"
|
||||
|
||||
|
|
@ -633,10 +639,25 @@ jobs:
|
|||
KEY = os.environ["API_KEY"]
|
||||
SEED = 3407
|
||||
|
||||
# ── 1. response_format = json_schema (strict JSON decoding) ──
|
||||
# Gemma 4 supports llama-server's GBNF-grammar-from-schema
|
||||
# decoding, so the response MUST be valid JSON matching the
|
||||
# schema even with a small model.
|
||||
def post(path, body, *, timeout = 240):
|
||||
req = urllib.request.Request(
|
||||
f"{BASE}{path}",
|
||||
data = json.dumps(body).encode(),
|
||||
method = "POST",
|
||||
headers = {
|
||||
"Authorization": f"Bearer {KEY}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout = timeout) as resp:
|
||||
return resp.status, json.loads(resp.read().decode())
|
||||
|
||||
# ── 1. response_format = json_object + schema (llama-server) ─
|
||||
# llama.cpp's HTTP server accepts the OpenAI "json_object"
|
||||
# mode plus an extension `schema` field that constrains the
|
||||
# output via a GBNF grammar derived from the JSON schema.
|
||||
# We use raw HTTP so there's no SDK ambiguity over which
|
||||
# response_format variant the SDK rewrites the field into.
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
|
@ -646,27 +667,28 @@ jobs:
|
|||
"required": ["city", "country"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
|
||||
resp = client.chat.completions.create(
|
||||
model = "default",
|
||||
messages = [
|
||||
status, data = post("/v1/chat/completions", {
|
||||
"model": "default",
|
||||
"messages": [
|
||||
{"role": "system", "content": "Reply with a single JSON object."},
|
||||
{"role": "user", "content": "What is the capital of France? Reply as JSON with city and country."},
|
||||
],
|
||||
temperature = 0.0,
|
||||
max_tokens = 80,
|
||||
seed = SEED,
|
||||
response_format = {
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "capital", "schema": schema, "strict": True},
|
||||
"temperature": 0.0,
|
||||
"max_tokens": 120,
|
||||
"seed": SEED,
|
||||
"stream": False,
|
||||
"enable_thinking": False,
|
||||
"response_format": {
|
||||
"type": "json_object",
|
||||
"schema": schema,
|
||||
},
|
||||
extra_body = {"enable_thinking": False},
|
||||
)
|
||||
content = resp.choices[0].message.content
|
||||
})
|
||||
assert status == 200, f"json status {status}: {data}"
|
||||
content = data["choices"][0]["message"]["content"] or ""
|
||||
parsed = json.loads(content)
|
||||
assert parsed.keys() == {"city", "country"}, f"schema mismatch: {parsed}"
|
||||
assert set(parsed.keys()) == {"city", "country"}, f"schema mismatch: {parsed}"
|
||||
assert "paris" in parsed["city"].lower(), f"city != Paris: {parsed}"
|
||||
print(f"[json] PASS strict json_schema -> {parsed}")
|
||||
print(f"[json] PASS schema-constrained json_object -> {parsed}")
|
||||
|
||||
# ── 2. OpenAI image_url (data URI base64) ───────────────────
|
||||
# 4x4 solid-red PNG. Tiny so the prompt fits in context. The
|
||||
|
|
@ -679,6 +701,7 @@ jobs:
|
|||
)
|
||||
data_uri = f"data:image/png;base64,{PNG_4X4_RED_B64}"
|
||||
|
||||
client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY)
|
||||
openai_resp = client.chat.completions.create(
|
||||
model = "default",
|
||||
temperature = 0.0,
|
||||
|
|
@ -701,7 +724,9 @@ jobs:
|
|||
print("[image/openai] PASS image_url accepted, non-empty response")
|
||||
|
||||
# ── 3. Anthropic source/base64 image ────────────────────────
|
||||
anthropic = Anthropic(base_url = f"{BASE}/v1", api_key = KEY)
|
||||
# Anthropic SDK appends /v1/messages itself; base_url is the
|
||||
# bare host (no /v1) -- otherwise the SDK posts to /v1/v1/messages.
|
||||
anthropic = Anthropic(base_url = BASE, api_key = KEY)
|
||||
a_msg = anthropic.messages.create(
|
||||
model = "default",
|
||||
max_tokens = 80,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue