ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix hand-rolled JSON to match Pydantic exclude_none=True output

Browse source 
The old code used model_dump_json(exclude_none=True) which omits
finish_reason entirely when it is None. The hand-rolled suffix
was incorrectly including "finish_reason":null. Removed it so the
SSE output is byte-for-byte identical to the old Pydantic path.
This commit is contained in:
Daniel Han 2026-03-26 10:34:12 +00:00
commit 3b1a416cd4
1 changed files with 9 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

15
studio/backend/routes/inference.py
View file

@ -1134,9 +1134,10 @@ async def openai_chat_completions(
)
yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n"
# Pre-compute static JSON envelope for content tokens (hot path)
# Pre-compute static JSON envelope for content tokens (hot path).
# Must match model_dump_json(exclude_none=True): no finish_reason when null.
_chunk_prefix = f'{{"id":"{completion_id}","object":"chat.completion.chunk","created":{created},"model":{json.dumps(model_name)},"choices":[{{"index":0,"delta":{{"content":'
_chunk_suffix = '},"finish_reason":null}]}'
_chunk_suffix = '}}]}'
gen = gguf_generate_with_tools()
prev_text = ""
@ -1271,9 +1272,10 @@ async def openai_chat_completions(
)
yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n"
# Pre-compute static JSON envelope for content tokens (hot path)
# Pre-compute static JSON envelope for content tokens (hot path).
# Must match model_dump_json(exclude_none=True): no finish_reason when null.
_chunk_prefix = f'{{"id":"{completion_id}","object":"chat.completion.chunk","created":{created},"model":{json.dumps(model_name)},"choices":[{{"index":0,"delta":{{"content":'
_chunk_suffix = '},"finish_reason":null}]}'
_chunk_suffix = '}}]}'
gen = gguf_generate()
prev_text = ""
@ -1469,9 +1471,10 @@ async def openai_chat_completions(
yield f"data: {first_chunk.model_dump_json(exclude_none = True)}\n\n"
prev_text = ""
# Pre-compute static JSON envelope for content tokens (hot path)
# Pre-compute static JSON envelope for content tokens (hot path).
# Must match model_dump_json(exclude_none=True): no finish_reason when null.
_chunk_prefix = f'{{"id":"{completion_id}","object":"chat.completion.chunk","created":{created},"model":{json.dumps(model_name)},"choices":[{{"index":0,"delta":{{"content":'
_chunk_suffix = '},"finish_reason":null}]}'
_chunk_suffix = '}}]}'
gen = generate()
_disconnect_check_interval = 20