diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 1fe134c3f9..147174451e 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -307,6 +307,26 @@ def _native_linux_system_rocm_lib_dirs(binary_dir: str = "") -> "list[str]": _DEFAULT_MAX_TOKENS_FLOOR = 32768 _DEFAULT_FIRST_TOKEN_TIMEOUT_S = 1200.0 # 20 min + +def _finalize_reasoning_only_cumulative( + cumulative: str, reasoning_text: str, finish_reason: Optional[str], promote_reasoning_only: bool +) -> str: + """Close a live thinking block and promote it only after a clean stop. + + Local inference streams cumulative snapshots. Replacing ``...`` with + bare reasoning at EOF makes the final snapshot shorter, so suffix-based + route consumers drop the intended fallback. Keep the snapshot append-only. + A length-truncated thought is not a final answer, so close it without + promotion and let the client surface the ``length`` terminal state. Raw + consumers that do not split reasoning from visible content can disable the + fallback to avoid returning the same reasoning twice. + """ + visible_fallback = ( + reasoning_text if promote_reasoning_only and finish_reason != "length" else "" + ) + return cumulative + "" + visible_fallback + + # Only large streamed tool payloads get an early provisional card; render_html # is exempt because it needs immediate artifact feedback. _PROVISIONAL_ARGS_MIN_CHARS = 256 @@ -10556,6 +10576,7 @@ class LlamaCppBackend: reasoning_effort: Optional[str] = None, preserve_thinking: Optional[bool] = None, seed: Optional[int] = None, + promote_reasoning_only: bool = True, _allow_respawn_retry: bool = True, ) -> Generator[Union[str, dict], None, None]: """ @@ -10638,7 +10659,12 @@ class LlamaCppBackend: # model put its whole reply in reasoning # (e.g. Qwen3 always-think). Show it as # the main response, not a thinking block. - cumulative = reasoning_text + cumulative = _finalize_reasoning_only_cumulative( + cumulative, + reasoning_text, + _metadata_finish_reason, + promote_reasoning_only, + ) yield cumulative _stream_done = True break # exit inner while @@ -10735,6 +10761,7 @@ class LlamaCppBackend: reasoning_effort = reasoning_effort, preserve_thinking = preserve_thinking, seed = seed, + promote_reasoning_only = promote_reasoning_only, _allow_respawn_retry = False, ) return @@ -10776,6 +10803,7 @@ class LlamaCppBackend: confirm_tool_calls: bool = False, bypass_permissions: bool = False, permission_mode: Optional[str] = None, + promote_reasoning_only: bool = True, ) -> Generator[dict, None, None]: """ Agentic loop: let the model call tools, execute them, and continue. @@ -11118,7 +11146,12 @@ class LlamaCppBackend: ), } else: - cumulative_display = reasoning_accum + cumulative_display = _finalize_reasoning_only_cumulative( + cumulative_display, + reasoning_accum, + _iter_finish_reason, + promote_reasoning_only, + ) if not _suppress_visible_output: yield { "type": "content", @@ -11582,7 +11615,12 @@ class LlamaCppBackend: if _reasoning_started_at is not None and not _reasoning_summary_emitted: _reasoning_summary_emitted = True yield _reasoning_summary_event(_reasoning_started_at) - cumulative_display = reasoning_accum + cumulative_display = _finalize_reasoning_only_cumulative( + cumulative_display, + reasoning_accum, + _iter_finish_reason, + promote_reasoning_only, + ) if not _suppress_visible_output: yield { "type": "content", @@ -12146,7 +12184,12 @@ class LlamaCppBackend: "text": _strip_tool_markup(cumulative, final = True), } else: - cumulative = reasoning_text + cumulative = _finalize_reasoning_only_cumulative( + cumulative, + reasoning_text, + _metadata_finish_reason, + promote_reasoning_only, + ) yield {"type": "content", "text": cumulative} _stream_done = True break # exit inner while diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 445a26f04d..4e74e1d32d 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -13355,6 +13355,7 @@ async def anthropic_messages( disable_parallel_tool_use = _disable_parallel, bypass_permissions = bool(payload.bypass_permissions), permission_mode = getattr(payload, "permission_mode", None), + promote_reasoning_only = False, ) if payload.stream: @@ -13394,6 +13395,7 @@ async def anthropic_messages( max_tokens = payload.max_tokens, stop = stop, cancel_event = cancel_event, + promote_reasoning_only = False, ) if payload.stream: diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py index 9ccc3f44dd..621ac9aaca 100644 --- a/studio/backend/tests/test_anthropic_messages.py +++ b/studio/backend/tests/test_anthropic_messages.py @@ -68,16 +68,15 @@ def _emitter_client_text(events: list[str]) -> str: def test_anthropic_emitter_closes_reasoning_only_think_block(): - # A reasoning-only reply streams X live then shrinks to bare X at EOF. - # This emitter diffs cumulative snapshots and drops the shrink, so without a - # closing pass the client text would end on an unclosed . finish() - # must balance it. + # Anthropic asks the GGUF generator not to promote reasoning into a duplicate + # visible fallback, so its final cumulative snapshot only balances the block. emitter = AnthropicStreamEmitter() events = emitter.start("msg_1", "m") events += emitter.feed({"type": "content", "text": "The capital"}) events += emitter.feed({"type": "content", "text": "The capital of France is Paris."}) - # The generator's final bare-text shrink (dropped by the cumulative diff). - events += emitter.feed({"type": "content", "text": "The capital of France is Paris."}) + events += emitter.feed( + {"type": "content", "text": "The capital of France is Paris."} + ) events += emitter.finish() assert _emitter_client_text(events) == "The capital of France is Paris." @@ -1563,6 +1562,44 @@ class TestAnthropicMessagesToolRouting: assert entry["context_length"] == 2048 assert monitor.active_count() == 0 + @pytest.mark.parametrize("stream", [False, True]) + @pytest.mark.parametrize("with_tools", [False, True]) + def test_reasoning_only_output_is_not_duplicated(self, monkeypatch, stream, with_tools): + reasoning = "The capital of France is Paris." + + def _gen_plain(**kwargs): + assert kwargs["promote_reasoning_only"] is False + yield f"{reasoning}" + yield f"{reasoning}" + + def _gen_tools(**kwargs): + assert kwargs["promote_reasoning_only"] is False + yield {"type": "content", "text": f"{reasoning}"} + yield {"type": "content", "text": f"{reasoning}"} + + _mock_backend( + monkeypatch, + generate_chat_completion = _gen_plain, + generate_chat_completion_with_tools = _gen_tools, + ) + payload_fields = {"stream": stream} + if with_tools: + payload_fields.update( + { + "enable_tools": True, + "tools": [{"type": "web_search_20250305", "name": "web_search"}], + } + ) + payload = _basic_payload(**payload_fields) + + response = _drive(anthropic_messages(payload, request = self._Request(), current_subject = "t")) + if stream: + body = self._sse_blob(self._consume_response(response)) + assert body.count(reasoning) == 1 + else: + body = json.loads(response.body) + assert body["content"][0]["text"] == f"{reasoning}" + def test_tool_use_non_streaming_records_api_monitor_reply(self, monkeypatch): import routes.inference as inf_mod diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py index e99e227d40..cf9fde7118 100644 --- a/studio/backend/tests/test_llama_cpp_tool_loop.py +++ b/studio/backend/tests/test_llama_cpp_tool_loop.py @@ -37,6 +37,24 @@ def _done() -> str: return "data: [DONE]\n" +def _finish(reason: str) -> str: + return ( + "data: " + + json.dumps( + { + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": reason, + } + ] + } + ) + + "\n" + ) + + def _make_backend(monkeypatch, streams: list[list[str]], payloads: list[dict]): backend = LlamaCppBackend.__new__(LlamaCppBackend) backend._process = object() @@ -299,9 +317,8 @@ def test_reasoning_streams_incrementally_with_tools(monkeypatch): def test_reasoning_only_reply_matches_no_tool_path_with_tools(monkeypatch): # A reasoning-only turn (whole answer in reasoning_content, no content, no # tool) with a tool active streams the reasoning live, then resolves to the - # bare reasoning text -- identical to the no-tool generate_chat_completion - # path -- so the non-streaming drain still returns it as `content`, not an - # empty answer. + # same text on the visible channel. The final cumulative snapshot stays + # append-only so route suffix extraction cannot drop that fallback. stream = [ _sse({"reasoning_content": "The capital of France is Paris."}), _done(), @@ -321,8 +338,49 @@ def test_reasoning_only_reply_matches_no_tool_path_with_tools(monkeypatch): content_texts = [e["text"] for e in events if e["type"] == "content"] # Reasoning streamed live during BUFFERING (the fix). assert content_texts[0] == "The capital of France is Paris." - # Resolves to bare reasoning, matching the no-tool sibling. - assert content_texts[-1] == "The capital of France is Paris." + assert content_texts[-1] == ( + "The capital of France is Paris.The capital of France is Paris." + ) + + +def _assert_reasoning_only_raw_consumer_gets_one_balanced_think_block(monkeypatch, with_tools): + stream = [ + _sse({"reasoning_content": "The capital of France is Paris."}), + _done(), + ] + backend = _make_backend(monkeypatch, [stream], []) + + if with_tools: + items = list( + backend.generate_chat_completion_with_tools( + messages = [{"role": "user", "content": "capital of France?"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + max_tool_iterations = 1, + promote_reasoning_only = False, + ) + ) + cumulatives = [item["text"] for item in items if item.get("type") == "content"] + else: + items = list( + backend.generate_chat_completion( + messages = [{"role": "user", "content": "capital of France?"}], + promote_reasoning_only = False, + ) + ) + cumulatives = [item for item in items if isinstance(item, str)] + + assert cumulatives[-1] == "The capital of France is Paris." + assert all( + current.startswith(previous) for previous, current in zip([""] + cumulatives, cumulatives) + ) + + +def test_reasoning_only_raw_consumer_without_tools_gets_one_balanced_think_block(monkeypatch): + _assert_reasoning_only_raw_consumer_gets_one_balanced_think_block(monkeypatch, False) + + +def test_reasoning_only_raw_consumer_with_tools_gets_one_balanced_think_block(monkeypatch): + _assert_reasoning_only_raw_consumer_gets_one_balanced_think_block(monkeypatch, True) def test_reasoning_before_structured_tool_closes_think_block(monkeypatch): @@ -392,8 +450,8 @@ def _replay_route_reasoning_extractor(cumulatives: list[str]) -> tuple[str, str] def test_reasoning_only_route_output_matches_no_tool_path(monkeypatch): # Parity contract: a reasoning-only reply must reach the client identically # whether tools are on or off. Both generators stream live then - # resolve to the bare reasoning text; the route's suffix-diff + extractor - # must therefore produce the same (visible, reasoning) split for both. + # append a balanced close plus visible fallback; the route's suffix-diff + + # extractor must therefore produce the same split for both. stream = [ _sse({"reasoning_content": "The capital"}), _sse({"reasoning_content": " of France is Paris."}), @@ -430,10 +488,37 @@ def test_reasoning_only_route_output_matches_no_tool_path(monkeypatch): no_tool_out = _replay_route_reasoning_extractor(no_tool_cumulatives) assert tool_out == no_tool_out # Pin the shared contract so a change to either path shows up here. - _visible, reasoning = tool_out + visible, reasoning = tool_out + assert visible == "The capital of France is Paris." assert reasoning == "The capital of France is Paris." +def test_length_truncated_reasoning_stays_append_only_without_visible_promotion(monkeypatch): + stream = [ + _sse({"reasoning_content": "The proof begins by assuming finitely many primes."}), + _finish("length"), + _done(), + ] + backend = _make_backend(monkeypatch, [stream], []) + + items = list( + backend.generate_chat_completion( + messages = [{"role": "user", "content": "Prove infinitely many primes"}], + max_tokens = 16, + ) + ) + cumulatives = [item for item in items if isinstance(item, str)] + + assert all( + current.startswith(previous) for previous, current in zip([""] + cumulatives, cumulatives) + ) + assert cumulatives[-1] == ("The proof begins by assuming finitely many primes.") + visible, reasoning = _replay_route_reasoning_extractor(cumulatives) + assert visible == "" + assert reasoning == "The proof begins by assuming finitely many primes." + assert items[-1]["finish_reason"] == "length" + + def test_reasoning_before_bare_json_tool_closes_think_block(monkeypatch): # _drain_silently sibling of the structured-tool close: a bare-JSON tool call # with a live reasoning prefix must also close before draining, and diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index d4861e8a3a..b7323777b2 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -89,6 +89,7 @@ import { import { resolveLoadMaxSeqLength } from "../presets/preset-policy"; import { generateAudio, + GenerationLengthError, listCachedGguf, listCachedModels, listGgufVariants, @@ -4093,7 +4094,15 @@ export function createOpenAIStreamAdapter( ); if (!abortSignal.aborted) { const msg = err instanceof Error ? err.message : String(err); - if (err instanceof StreamInterruptedError) { + if (err instanceof GenerationLengthError) { + toast.error("Response ran out of tokens", { + description: + "The model used the full Max Tokens budget while thinking " + + "and did not produce a final answer. Increase Max Tokens in " + + "chat Settings or turn off thinking, then retry.", + duration: 8000, + }); + } else if (err instanceof StreamInterruptedError) { // Connection dropped mid-turn: surface it explicitly (the rethrow // below also marks the message with an inline error + Retry). toast.error("Response interrupted", { diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index ffaf099f29..4d123e98ab 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -50,6 +50,21 @@ export class StreamInterruptedError extends Error { } } +/** + * Thrown when a reasoning model consumes its output budget before emitting any + * standard content. Keeping this distinct from a dropped connection lets the + * chat UI explain why a completed stream contains only a thinking panel. + */ +export class GenerationLengthError extends Error { + constructor() { + super( + "The model reached the Max Tokens limit before producing a final answer. " + + "Increase Max Tokens or disable thinking, then retry.", + ); + this.name = "GenerationLengthError"; + } +} + export function notifyChatHistoryUpdated(): void { if (typeof window !== "undefined") { window.dispatchEvent(new Event(CHAT_HISTORY_UPDATED_EVENT)); @@ -982,6 +997,61 @@ function parseSseEvent(rawEvent: string): string[] { return dataLines; } +function hasNonWhitespaceText(value: unknown): boolean { + if (typeof value === "string") { + return value.trim().length > 0; + } + if (Array.isArray(value)) { + return value.some((item) => hasNonWhitespaceText(item)); + } + if (!value || typeof value !== "object") { + return false; + } + const record = value as Record; + return ["thinking", "text", "content", "reasoning", "summary"].some( + (key) => key in record && hasNonWhitespaceText(record[key]), + ); +} + +function classifyStructuredDeltaContent(content: unknown): { + hasAssistantContent: boolean; + hasReasoningContent: boolean; +} { + if (typeof content === "string") { + return { + hasAssistantContent: hasNonWhitespaceText(content), + hasReasoningContent: false, + }; + } + if (!Array.isArray(content)) { + return { + hasAssistantContent: false, + hasReasoningContent: false, + }; + } + + let hasAssistantContent = false; + let hasReasoningContent = false; + for (const part of content) { + if (typeof part === "string") { + hasAssistantContent ||= hasNonWhitespaceText(part); + continue; + } + if (!part || typeof part !== "object") { + continue; + } + const record = part as Record; + if (record.type === "thinking" || record.type === "reasoning") { + hasReasoningContent ||= hasNonWhitespaceText(record); + } else if (record.type === "text" || record.type === "output_text") { + const text = + typeof record.text === "string" ? record.text : record.content; + hasAssistantContent ||= hasNonWhitespaceText(text); + } + } + return { hasAssistantContent, hasReasoningContent }; +} + export async function* streamChatCompletions( payload: OpenAIChatCompletionsRequest, signal: AbortSignal, @@ -1009,6 +1079,19 @@ export async function* streamChatCompletions( // EOF without `[DONE]` or a finish_reason chunk means the stream was cut // mid-generation: surface as interrupted, not silent success. let sawTerminalSignal = false; + let terminalFinishReason: string | null = null; + let sawAssistantContent = false; + let sawReasoningContent = false; + + const throwIfReasoningOnlyLength = () => { + if ( + terminalFinishReason === "length" && + sawReasoningContent && + !sawAssistantContent + ) { + throw new GenerationLengthError(); + } + }; try { while (true) { @@ -1018,6 +1101,7 @@ export async function* streamChatCompletions( if (!sawTerminalSignal) { throw new StreamInterruptedError(); } + throwIfReasoningOnlyLength(); break; } @@ -1039,6 +1123,7 @@ export async function* streamChatCompletions( if (dataText === "[DONE]") { completed = true; sawTerminalSignal = true; + throwIfReasoningOnlyLength(); return; } @@ -1094,11 +1179,31 @@ export async function* streamChatCompletions( } // finish_reason is a valid terminal signal for providers that close // the stream without an explicit [DONE] sentinel. - const finishReason = ( + const parsedChoices = ( parsed as { - choices?: Array<{ finish_reason?: string | null }>; + choices?: Array<{ + delta?: Record; + finish_reason?: string | null; + }>; } - ).choices?.[0]?.finish_reason; + ).choices; + for (const choice of parsedChoices ?? []) { + const delta = choice.delta; + if (delta) { + const contentState = classifyStructuredDeltaContent(delta.content); + sawAssistantContent ||= contentState.hasAssistantContent; + sawReasoningContent ||= contentState.hasReasoningContent; + const reasoning = + delta.reasoning_content ?? + delta.reasoning ?? + delta.reasoning_details; + sawReasoningContent ||= hasNonWhitespaceText(reasoning); + } + if (choice.finish_reason) { + terminalFinishReason = choice.finish_reason; + } + } + const finishReason = parsedChoices?.[0]?.finish_reason; if (finishReason) { sawTerminalSignal = true; } diff --git a/tests/studio/test_generation_length_ui_contract.py b/tests/studio/test_generation_length_ui_contract.py new file mode 100644 index 0000000000..cc25f91080 --- /dev/null +++ b/tests/studio/test_generation_length_ui_contract.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. + +from pathlib import Path + + +CHAT_API = ( + Path(__file__).resolve().parents[2] + / "studio" + / "frontend" + / "src" + / "features" + / "chat" + / "api" + / "chat-api.ts" +) + + +def test_length_detection_classifies_visible_and_reasoning_content(): + source = CHAT_API.read_text(encoding = "utf-8") + + assert "return value.trim().length > 0;" in source + assert 'record.type === "thinking" || record.type === "reasoning"' in source + assert 'record.type === "text" || record.type === "output_text"' in source + assert "sawAssistantContent ||= contentState.hasAssistantContent;" in source + assert "sawReasoningContent ||= contentState.hasReasoningContent;" in source