From bb1f4ed5dc81aad0fdf55ff65f95f2f24ed4c9db Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 6 May 2026 12:36:47 +0000 Subject: [PATCH] CI: fix three regressions in the new Studio GGUF jobs Job 1 (OpenAI, Anthropic API tests): Anthropic SDK appends /v1/messages to base_url itself, so passing base_url=f"{BASE}/v1" produced /v1/v1/messages and 405'd. Bare BASE is correct (matches the docs' "the SDK appends /v1 automatically"). OpenAI SDK side already worked: 4-turn transcript was fully deterministic across two runs and the "Paris" sanity assertion passed. Job 2 (tool calling tests): Booting with --enable-tools forces the process-level tool policy to True for every request (state/tool_policy.py:get_tool_policy), which hijacked the "Standard OpenAI function calling" test through the server-side agentic loop -- the model called web_search instead of returning structured tool_calls for the user's `weather_tool`. Drop --enable-tools so policy is None (per-request honour). The python / terminal / web_search probes already pass enable_tools=True explicitly in their request bodies, so they keep working. Job 3 (JSON, images): Two issues. (a) The OpenAI Python SDK rewrites response_format={"type":"json_schema",...} into something Studio's llama-server backend doesn't accept, so resp came back as the raw error string and resp.choices[0] tripped 'str has no attribute choices'. Switched to raw HTTP with the `{"type":"json_object", "schema":...}` form llama-server actually supports (GBNF-from-schema, llama-server extension). (b) Anthropic SDK base_url same fix as job 1. --- .github/workflows/studio-inference-smoke.yml | 75 +++++++++++++------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/.github/workflows/studio-inference-smoke.yml b/.github/workflows/studio-inference-smoke.yml index 4b88e12af5..7fb9ab98e2 100644 --- a/.github/workflows/studio-inference-smoke.yml +++ b/.github/workflows/studio-inference-smoke.yml @@ -213,7 +213,10 @@ jobs: return replies def run_anthropic(): - client = Anthropic(base_url = f"{BASE}/v1", api_key = KEY) + # Anthropic SDK appends /v1/messages itself, so base_url + # must NOT include /v1 (otherwise the request hits + # /v1/v1/messages and 405s). + client = Anthropic(base_url = BASE, api_key = KEY) history, replies = [], [] for prompt in PROMPTS: history.append({"role": "user", "content": prompt}) @@ -325,13 +328,16 @@ jobs: mkdir -p logs # `unsloth studio run` boots the server, loads the GGUF via # the HF_HOME-cached path, and prints the API key on the - # banner. --enable-tools makes the server-side tool registry - # (python / terminal / web_search) available behind the - # enable_tools=true request flag. + # banner. We deliberately do NOT pass --enable-tools: that + # forces the process-level tool policy to True, which + # hijacks every request through the server-side agentic + # loop and breaks the function-calling test below. Default + # policy (None) honours each request's `enable_tools` flag, + # which is what the script under "Tool calling" relies on. unsloth studio run \ --model "$GGUF_REPO" --gguf-variant "$GGUF_VARIANT" \ --port "$STUDIO_PORT" --host 127.0.0.1 \ - --enable-tools -y \ + -y \ > logs/studio.log 2>&1 & echo "STUDIO_PID=$!" >> "$GITHUB_ENV" @@ -633,10 +639,25 @@ jobs: KEY = os.environ["API_KEY"] SEED = 3407 - # ── 1. response_format = json_schema (strict JSON decoding) ── - # Gemma 4 supports llama-server's GBNF-grammar-from-schema - # decoding, so the response MUST be valid JSON matching the - # schema even with a small model. + def post(path, body, *, timeout = 240): + req = urllib.request.Request( + f"{BASE}{path}", + data = json.dumps(body).encode(), + method = "POST", + headers = { + "Authorization": f"Bearer {KEY}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(req, timeout = timeout) as resp: + return resp.status, json.loads(resp.read().decode()) + + # ── 1. response_format = json_object + schema (llama-server) ─ + # llama.cpp's HTTP server accepts the OpenAI "json_object" + # mode plus an extension `schema` field that constrains the + # output via a GBNF grammar derived from the JSON schema. + # We use raw HTTP so there's no SDK ambiguity over which + # response_format variant the SDK rewrites the field into. schema = { "type": "object", "properties": { @@ -646,27 +667,28 @@ jobs: "required": ["city", "country"], "additionalProperties": False, } - client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY) - resp = client.chat.completions.create( - model = "default", - messages = [ + status, data = post("/v1/chat/completions", { + "model": "default", + "messages": [ {"role": "system", "content": "Reply with a single JSON object."}, {"role": "user", "content": "What is the capital of France? Reply as JSON with city and country."}, ], - temperature = 0.0, - max_tokens = 80, - seed = SEED, - response_format = { - "type": "json_schema", - "json_schema": {"name": "capital", "schema": schema, "strict": True}, + "temperature": 0.0, + "max_tokens": 120, + "seed": SEED, + "stream": False, + "enable_thinking": False, + "response_format": { + "type": "json_object", + "schema": schema, }, - extra_body = {"enable_thinking": False}, - ) - content = resp.choices[0].message.content + }) + assert status == 200, f"json status {status}: {data}" + content = data["choices"][0]["message"]["content"] or "" parsed = json.loads(content) - assert parsed.keys() == {"city", "country"}, f"schema mismatch: {parsed}" + assert set(parsed.keys()) == {"city", "country"}, f"schema mismatch: {parsed}" assert "paris" in parsed["city"].lower(), f"city != Paris: {parsed}" - print(f"[json] PASS strict json_schema -> {parsed}") + print(f"[json] PASS schema-constrained json_object -> {parsed}") # ── 2. OpenAI image_url (data URI base64) ─────────────────── # 4x4 solid-red PNG. Tiny so the prompt fits in context. The @@ -679,6 +701,7 @@ jobs: ) data_uri = f"data:image/png;base64,{PNG_4X4_RED_B64}" + client = OpenAI(base_url = f"{BASE}/v1", api_key = KEY) openai_resp = client.chat.completions.create( model = "default", temperature = 0.0, @@ -701,7 +724,9 @@ jobs: print("[image/openai] PASS image_url accepted, non-empty response") # ── 3. Anthropic source/base64 image ──────────────────────── - anthropic = Anthropic(base_url = f"{BASE}/v1", api_key = KEY) + # Anthropic SDK appends /v1/messages itself; base_url is the + # bare host (no /v1) -- otherwise the SDK posts to /v1/v1/messages. + anthropic = Anthropic(base_url = BASE, api_key = KEY) a_msg = anthropic.messages.create( model = "default", max_tokens = 80,