ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: prevent empty responses after model thinking (#7418)

Browse source 
* Fix reasoning-only Qwen3.6 completions in Studio

* Address reasoning-only review findings

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
oobabooga 2026-07-24 21:01:12 -03:00 • committed by GitHub
commit 91a89806d7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 329 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

51
studio/backend/core/inference/llama_cpp.py
View file

@ -307,6 +307,26 @@ def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> "list[str]":
_DEFAULT_MAX_TOKENS_FLOOR = 32768
_DEFAULT_FIRST_TOKEN_TIMEOUT_S = 1200.0 # 20 min
def _finalize_reasoning_only_cumulative(
cumulative: str, reasoning_text: str, finish_reason: Optional[str], promote_reasoning_only: bool
) -> str:
"""Close a live thinking block and promote it only after a clean stop.
Local inference streams cumulative snapshots. Replacing ``<think>...`` with
bare reasoning at EOF makes the final snapshot shorter, so suffix-based
route consumers drop the intended fallback. Keep the snapshot append-only.
A length-truncated thought is not a final answer, so close it without
promotion and let the client surface the ``length`` terminal state. Raw
consumers that do not split reasoning from visible content can disable the
fallback to avoid returning the same reasoning twice.
"""
visible_fallback = (
reasoning_text if promote_reasoning_only and finish_reason != "length" else ""
)
return cumulative + "</think>" + visible_fallback
# Only large streamed tool payloads get an early provisional card; render_html
# is exempt because it needs immediate artifact feedback.
_PROVISIONAL_ARGS_MIN_CHARS = 256
@ -10556,6 +10576,7 @@ class LlamaCppBackend:
reasoning_effort: Optional[str] = None,
preserve_thinking: Optional[bool] = None,
seed: Optional[int] = None,
promote_reasoning_only: bool = True,
_allow_respawn_retry: bool = True,
) -> Generator[Union[str, dict], None, None]:
"""
@ -10638,7 +10659,12 @@ class LlamaCppBackend:
# model put its whole reply in reasoning
# (e.g. Qwen3 always-think). Show it as
# the main response, not a thinking block.
cumulative = reasoning_text
cumulative = _finalize_reasoning_only_cumulative(
cumulative,
reasoning_text,
_metadata_finish_reason,
promote_reasoning_only,
)
yield cumulative
_stream_done = True
break # exit inner while
@ -10735,6 +10761,7 @@ class LlamaCppBackend:
reasoning_effort = reasoning_effort,
preserve_thinking = preserve_thinking,
seed = seed,
promote_reasoning_only = promote_reasoning_only,
_allow_respawn_retry = False,
)
return
@ -10776,6 +10803,7 @@ class LlamaCppBackend:
confirm_tool_calls: bool = False,
bypass_permissions: bool = False,
permission_mode: Optional[str] = None,
promote_reasoning_only: bool = True,
) -> Generator[dict, None, None]:
"""
Agentic loop: let the model call tools, execute them, and continue.
@ -11118,7 +11146,12 @@ class LlamaCppBackend:
),
}
else:
cumulative_display = reasoning_accum
cumulative_display = _finalize_reasoning_only_cumulative(
cumulative_display,
reasoning_accum,
_iter_finish_reason,
promote_reasoning_only,
)
if not _suppress_visible_output:
yield {
"type": "content",
@ -11582,7 +11615,12 @@ class LlamaCppBackend:
if _reasoning_started_at is not None and not _reasoning_summary_emitted:
_reasoning_summary_emitted = True
yield _reasoning_summary_event(_reasoning_started_at)
cumulative_display = reasoning_accum
cumulative_display = _finalize_reasoning_only_cumulative(
cumulative_display,
reasoning_accum,
_iter_finish_reason,
promote_reasoning_only,
)
if not _suppress_visible_output:
yield {
"type": "content",
@ -12146,7 +12184,12 @@ class LlamaCppBackend:
"text": _strip_tool_markup(cumulative, final = True),
}
else:
cumulative = reasoning_text
cumulative = _finalize_reasoning_only_cumulative(
cumulative,
reasoning_text,
_metadata_finish_reason,
promote_reasoning_only,
)
yield {"type": "content", "text": cumulative}
_stream_done = True
break # exit inner while

2
studio/backend/routes/inference.py
View file

@ -13355,6 +13355,7 @@ async def anthropic_messages(
disable_parallel_tool_use = _disable_parallel,
bypass_permissions = bool(payload.bypass_permissions),
permission_mode = getattr(payload, "permission_mode", None),
promote_reasoning_only = False,
)
if payload.stream:
@ -13394,6 +13395,7 @@ async def anthropic_messages(
max_tokens = payload.max_tokens,
stop = stop,
cancel_event = cancel_event,
promote_reasoning_only = False,
)
if payload.stream:

49
studio/backend/tests/test_anthropic_messages.py
View file

@ -68,16 +68,15 @@ def _emitter_client_text(events: list[str]) -> str:
def test_anthropic_emitter_closes_reasoning_only_think_block():
# A reasoning-only reply streams <think>X live then shrinks to bare X at EOF.
# This emitter diffs cumulative snapshots and drops the shrink, so without a
# closing pass the client text would end on an unclosed <think>. finish()
# must balance it.
# Anthropic asks the GGUF generator not to promote reasoning into a duplicate
# visible fallback, so its final cumulative snapshot only balances the block.
emitter = AnthropicStreamEmitter()
events = emitter.start("msg_1", "m")
events += emitter.feed({"type": "content", "text": "<think>The capital"})
events += emitter.feed({"type": "content", "text": "<think>The capital of France is Paris."})
# The generator's final bare-text shrink (dropped by the cumulative diff).
events += emitter.feed({"type": "content", "text": "The capital of France is Paris."})
events += emitter.feed(
{"type": "content", "text": "<think>The capital of France is Paris.</think>"}
)
events += emitter.finish()
assert _emitter_client_text(events) == "<think>The capital of France is Paris.</think>"
@ -1563,6 +1562,44 @@ class TestAnthropicMessagesToolRouting:
assert entry["context_length"] == 2048
assert monitor.active_count() == 0
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("with_tools", [False, True])
def test_reasoning_only_output_is_not_duplicated(self, monkeypatch, stream, with_tools):
reasoning = "The capital of France is Paris."
def _gen_plain(**kwargs):
assert kwargs["promote_reasoning_only"] is False
yield f"<think>{reasoning}"
yield f"<think>{reasoning}</think>"
def _gen_tools(**kwargs):
assert kwargs["promote_reasoning_only"] is False
yield {"type": "content", "text": f"<think>{reasoning}"}
yield {"type": "content", "text": f"<think>{reasoning}</think>"}
_mock_backend(
monkeypatch,
generate_chat_completion = _gen_plain,
generate_chat_completion_with_tools = _gen_tools,
)
payload_fields = {"stream": stream}
if with_tools:
payload_fields.update(
{
"enable_tools": True,
"tools": [{"type": "web_search_20250305", "name": "web_search"}],
}
)
payload = _basic_payload(**payload_fields)
response = _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t"))
if stream:
body = self._sse_blob(self._consume_response(response))
assert body.count(reasoning) == 1
else:
body = json.loads(response.body)
assert body["content"][0]["text"] == f"<think>{reasoning}</think>"
def test_tool_use_non_streaming_records_api_monitor_reply(self, monkeypatch):
import routes.inference as inf_mod

101
studio/backend/tests/test_llama_cpp_tool_loop.py
View file

@ -37,6 +37,24 @@ def _done() -> str:
return "data: [DONE]\n"
def _finish(reason: str) -> str:
return (
"data: "
+ json.dumps(
{
"choices": [
{
"index": 0,
"delta": {},
"finish_reason": reason,
}
]
}
)
+ "\n"
)
def _make_backend(monkeypatch, streams: list[list[str]], payloads: list[dict]):
backend = LlamaCppBackend.__new__(LlamaCppBackend)
backend._process = object()
@ -299,9 +317,8 @@ def test_reasoning_streams_incrementally_with_tools(monkeypatch):
def test_reasoning_only_reply_matches_no_tool_path_with_tools(monkeypatch):
# A reasoning-only turn (whole answer in reasoning_content, no content, no
# tool) with a tool active streams the reasoning live, then resolves to the
# bare reasoning text -- identical to the no-tool generate_chat_completion
# path -- so the non-streaming drain still returns it as `content`, not an
# empty answer.
# same text on the visible channel. The final cumulative snapshot stays
# append-only so route suffix extraction cannot drop that fallback.
stream = [
_sse({"reasoning_content": "The capital of France is Paris."}),
_done(),
@ -321,8 +338,49 @@ def test_reasoning_only_reply_matches_no_tool_path_with_tools(monkeypatch):
content_texts = [e["text"] for e in events if e["type"] == "content"]
# Reasoning streamed live during BUFFERING (the fix).
assert content_texts[0] == "<think>The capital of France is Paris."
# Resolves to bare reasoning, matching the no-tool sibling.
assert content_texts[-1] == "The capital of France is Paris."
assert content_texts[-1] == (
"<think>The capital of France is Paris.</think>The capital of France is Paris."
)
def _assert_reasoning_only_raw_consumer_gets_one_balanced_think_block(monkeypatch, with_tools):
stream = [
_sse({"reasoning_content": "The capital of France is Paris."}),
_done(),
]
backend = _make_backend(monkeypatch, [stream], [])
if with_tools:
items = list(
backend.generate_chat_completion_with_tools(
messages = [{"role": "user", "content": "capital of France?"}],
tools = [{"type": "function", "function": {"name": "web_search"}}],
max_tool_iterations = 1,
promote_reasoning_only = False,
)
)
cumulatives = [item["text"] for item in items if item.get("type") == "content"]
else:
items = list(
backend.generate_chat_completion(
messages = [{"role": "user", "content": "capital of France?"}],
promote_reasoning_only = False,
)
)
cumulatives = [item for item in items if isinstance(item, str)]
assert cumulatives[-1] == "<think>The capital of France is Paris.</think>"
assert all(
current.startswith(previous) for previous, current in zip([""] + cumulatives, cumulatives)
)
def test_reasoning_only_raw_consumer_without_tools_gets_one_balanced_think_block(monkeypatch):
_assert_reasoning_only_raw_consumer_gets_one_balanced_think_block(monkeypatch, False)
def test_reasoning_only_raw_consumer_with_tools_gets_one_balanced_think_block(monkeypatch):
_assert_reasoning_only_raw_consumer_gets_one_balanced_think_block(monkeypatch, True)
def test_reasoning_before_structured_tool_closes_think_block(monkeypatch):
@ -392,8 +450,8 @@ def _replay_route_reasoning_extractor(cumulatives: list[str]) -> tuple[str, str]
def test_reasoning_only_route_output_matches_no_tool_path(monkeypatch):
# Parity contract: a reasoning-only reply must reach the client identically
# whether tools are on or off. Both generators stream <think> live then
# resolve to the bare reasoning text; the route's suffix-diff + extractor
# must therefore produce the same (visible, reasoning) split for both.
# append a balanced close plus visible fallback; the route's suffix-diff +
# extractor must therefore produce the same split for both.
stream = [
_sse({"reasoning_content": "The capital"}),
_sse({"reasoning_content": " of France is Paris."}),
@ -430,10 +488,37 @@ def test_reasoning_only_route_output_matches_no_tool_path(monkeypatch):
no_tool_out = _replay_route_reasoning_extractor(no_tool_cumulatives)
assert tool_out == no_tool_out
# Pin the shared contract so a change to either path shows up here.
_visible, reasoning = tool_out
visible, reasoning = tool_out
assert visible == "The capital of France is Paris."
assert reasoning == "The capital of France is Paris."
def test_length_truncated_reasoning_stays_append_only_without_visible_promotion(monkeypatch):
stream = [
_sse({"reasoning_content": "The proof begins by assuming finitely many primes."}),
_finish("length"),
_done(),
]
backend = _make_backend(monkeypatch, [stream], [])
items = list(
backend.generate_chat_completion(
messages = [{"role": "user", "content": "Prove infinitely many primes"}],
max_tokens = 16,
)
)
cumulatives = [item for item in items if isinstance(item, str)]
assert all(
current.startswith(previous) for previous, current in zip([""] + cumulatives, cumulatives)
)
assert cumulatives[-1] == ("<think>The proof begins by assuming finitely many primes.</think>")
visible, reasoning = _replay_route_reasoning_extractor(cumulatives)
assert visible == ""
assert reasoning == "The proof begins by assuming finitely many primes."
assert items[-1]["finish_reason"] == "length"
def test_reasoning_before_bare_json_tool_closes_think_block(monkeypatch):
# _drain_silently sibling of the structured-tool close: a bare-JSON tool call
# with a live reasoning prefix must also close </think> before draining, and

11
studio/frontend/src/features/chat/api/chat-adapter.ts
View file

@ -89,6 +89,7 @@ import {
import { resolveLoadMaxSeqLength } from "../presets/preset-policy";
import {
generateAudio,
GenerationLengthError,
listCachedGguf,
listCachedModels,
listGgufVariants,
@ -4093,7 +4094,15 @@ export function createOpenAIStreamAdapter(
);
if (!abortSignal.aborted) {
const msg = err instanceof Error ? err.message : String(err);
if (err instanceof StreamInterruptedError) {
if (err instanceof GenerationLengthError) {
toast.error("Response ran out of tokens", {
description:
"The model used the full Max Tokens budget while thinking " +
"and did not produce a final answer. Increase Max Tokens in " +
"chat Settings or turn off thinking, then retry.",
duration: 8000,
});
} else if (err instanceof StreamInterruptedError) {
// Connection dropped mid-turn: surface it explicitly (the rethrow
// below also marks the message with an inline error + Retry).
toast.error("Response interrupted", {

111
studio/frontend/src/features/chat/api/chat-api.ts
View file

@ -50,6 +50,21 @@ export class StreamInterruptedError extends Error {
}
}
/**
* Thrown when a reasoning model consumes its output budget before emitting any
* standard content. Keeping this distinct from a dropped connection lets the
* chat UI explain why a completed stream contains only a thinking panel.
*/
export class GenerationLengthError extends Error {
constructor() {
super(
"The model reached the Max Tokens limit before producing a final answer. " +
"Increase Max Tokens or disable thinking, then retry.",
);
this.name = "GenerationLengthError";
}
}
export function notifyChatHistoryUpdated(): void {
if (typeof window !== "undefined") {
window.dispatchEvent(new Event(CHAT_HISTORY_UPDATED_EVENT));
@ -982,6 +997,61 @@ function parseSseEvent(rawEvent: string): string[] {
return dataLines;
}
function hasNonWhitespaceText(value: unknown): boolean {
if (typeof value === "string") {
return value.trim().length > 0;
}
if (Array.isArray(value)) {
return value.some((item) => hasNonWhitespaceText(item));
}
if (!value || typeof value !== "object") {
return false;
}
const record = value as Record<string, unknown>;
return ["thinking", "text", "content", "reasoning", "summary"].some(
(key) => key in record && hasNonWhitespaceText(record[key]),
);
}
function classifyStructuredDeltaContent(content: unknown): {
hasAssistantContent: boolean;
hasReasoningContent: boolean;
} {
if (typeof content === "string") {
return {
hasAssistantContent: hasNonWhitespaceText(content),
hasReasoningContent: false,
};
}
if (!Array.isArray(content)) {
return {
hasAssistantContent: false,
hasReasoningContent: false,
};
}
let hasAssistantContent = false;
let hasReasoningContent = false;
for (const part of content) {
if (typeof part === "string") {
hasAssistantContent ||= hasNonWhitespaceText(part);
continue;
}
if (!part || typeof part !== "object") {
continue;
}
const record = part as Record<string, unknown>;
if (record.type === "thinking" || record.type === "reasoning") {
hasReasoningContent ||= hasNonWhitespaceText(record);
} else if (record.type === "text" || record.type === "output_text") {
const text =
typeof record.text === "string" ? record.text : record.content;
hasAssistantContent ||= hasNonWhitespaceText(text);
}
}
return { hasAssistantContent, hasReasoningContent };
}
export async function* streamChatCompletions(
payload: OpenAIChatCompletionsRequest,
signal: AbortSignal,
@ -1009,6 +1079,19 @@ export async function* streamChatCompletions(
// EOF without `[DONE]` or a finish_reason chunk means the stream was cut
// mid-generation: surface as interrupted, not silent success.
let sawTerminalSignal = false;
let terminalFinishReason: string | null = null;
let sawAssistantContent = false;
let sawReasoningContent = false;
const throwIfReasoningOnlyLength = () => {
if (
terminalFinishReason === "length" &&
sawReasoningContent &&
!sawAssistantContent
) {
throw new GenerationLengthError();
}
};
try {
while (true) {
@ -1018,6 +1101,7 @@ export async function* streamChatCompletions(
if (!sawTerminalSignal) {
throw new StreamInterruptedError();
}
throwIfReasoningOnlyLength();
break;
}
@ -1039,6 +1123,7 @@ export async function* streamChatCompletions(
if (dataText === "[DONE]") {
completed = true;
sawTerminalSignal = true;
throwIfReasoningOnlyLength();
return;
}
@ -1094,11 +1179,31 @@ export async function* streamChatCompletions(
}
// finish_reason is a valid terminal signal for providers that close
// the stream without an explicit [DONE] sentinel.
const finishReason = (
const parsedChoices = (
parsed as {
choices?: Array<{ finish_reason?: string | null }>;
choices?: Array<{
delta?: Record<string, unknown>;
finish_reason?: string | null;
}>;
}
).choices?.[0]?.finish_reason;
).choices;
for (const choice of parsedChoices ?? []) {
const delta = choice.delta;
if (delta) {
const contentState = classifyStructuredDeltaContent(delta.content);
sawAssistantContent ||= contentState.hasAssistantContent;
sawReasoningContent ||= contentState.hasReasoningContent;
const reasoning =
delta.reasoning_content ??
delta.reasoning ??
delta.reasoning_details;
sawReasoningContent ||= hasNonWhitespaceText(reasoning);
}
if (choice.finish_reason) {
terminalFinishReason = choice.finish_reason;
}
}
const finishReason = parsedChoices?.[0]?.finish_reason;
if (finishReason) {
sawTerminalSignal = true;
}

26
tests/studio/test_generation_length_ui_contract.py Normal file
View file

@ -0,0 +1,26 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
from pathlib import Path
CHAT_API = (
Path(__file__).resolve().parents[2]
/ "studio"
/ "frontend"
/ "src"
/ "features"
/ "chat"
/ "api"
/ "chat-api.ts"
)
def test_length_detection_classifies_visible_and_reasoning_content():
source = CHAT_API.read_text(encoding = "utf-8")
assert "return value.trim().length > 0;" in source
assert 'record.type === "thinking" || record.type === "reasoning"' in source
assert 'record.type === "text" || record.type === "output_text"' in source
assert "sawAssistantContent ||= contentState.hasAssistantContent;" in source
assert "sawReasoningContent ||= contentState.hasReasoningContent;" in source