From 3b1a416cd4393b0e69111bfe0f191360e26c3eb8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 26 Mar 2026 10:34:12 +0000 Subject: [PATCH] Fix hand-rolled JSON to match Pydantic exclude_none=True output The old code used model_dump_json(exclude_none=True) which omits finish_reason entirely when it is None. The hand-rolled suffix was incorrectly including "finish_reason":null. Removed it so the SSE output is byte-for-byte identical to the old Pydantic path. --- studio/backend/routes/inference.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 3d59a70b6f..a71fd1b8cc 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1134,9 +1134,10 @@ async def openai_chat_completions( ) yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" - # Pre-compute static JSON envelope for content tokens (hot path) + # Pre-compute static JSON envelope for content tokens (hot path). + # Must match model_dump_json(exclude_none=True): no finish_reason when null. _chunk_prefix = f'{{"id":"{completion_id}","object":"chat.completion.chunk","created":{created},"model":{json.dumps(model_name)},"choices":[{{"index":0,"delta":{{"content":' - _chunk_suffix = '},"finish_reason":null}]}' + _chunk_suffix = '}}]}' gen = gguf_generate_with_tools() prev_text = "" @@ -1271,9 +1272,10 @@ async def openai_chat_completions( ) yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" - # Pre-compute static JSON envelope for content tokens (hot path) + # Pre-compute static JSON envelope for content tokens (hot path). + # Must match model_dump_json(exclude_none=True): no finish_reason when null. _chunk_prefix = f'{{"id":"{completion_id}","object":"chat.completion.chunk","created":{created},"model":{json.dumps(model_name)},"choices":[{{"index":0,"delta":{{"content":' - _chunk_suffix = '},"finish_reason":null}]}' + _chunk_suffix = '}}]}' gen = gguf_generate() prev_text = "" @@ -1469,9 +1471,10 @@ async def openai_chat_completions( yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n" prev_text = "" - # Pre-compute static JSON envelope for content tokens (hot path) + # Pre-compute static JSON envelope for content tokens (hot path). + # Must match model_dump_json(exclude_none=True): no finish_reason when null. _chunk_prefix = f'{{"id":"{completion_id}","object":"chat.completion.chunk","created":{created},"model":{json.dumps(model_name)},"choices":[{{"index":0,"delta":{{"content":' - _chunk_suffix = '},"finish_reason":null}]}' + _chunk_suffix = '}}]}' gen = generate() _disconnect_check_interval = 20