From 6866362da7cfd9471729ac1413fab24e2e5596c4 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 23 Jun 2026 05:59:56 -0700 Subject: [PATCH] studio: report the true reasoning duration and fix Stop for thinking models (#6521) * studio: report the true reasoning duration and fix the Stop button for thinking models For a local GGUF the "Thought for N" label was timed entirely on the client by a brittle edge-detector, so an always-think model (Qwen3 MTP) that buffers its whole reasoning and flushes it in one chunk showed "1 second" instead of the real minute-plus. The client cannot time reasoning it receives atomically, so make the timing backend-authoritative. Backend: generate_chat_completion_with_tools measures wall-clock reasoning and emits a Studio reasoning_summary event (duration_ms) at the moment reasoning ends -- the first answer token, or end-of-stream for a reasoning-only reply -- for both the tool-detection pass and the final-answer pass. Timing resets per tool iteration so the final answer's thinking time wins on the client (which takes the latest reasoning_summary). routes/inference.py forwards the event in the GGUF tool stream. Frontend: parse the reasoning_summary SSE into a _reasoningDurationMs chunk and use it as the authoritative reasoning duration (last write wins), clamped to >= 0 and guarded to a finite number so a malformed or proxied chunk cannot produce a NaN label; the persisted value wins for the final "Thought for N" label, with the previous live timer kept only as a fallback when no metadata arrives. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/core/inference/llama_cpp.py | 44 ++++++++- studio/backend/routes/inference.py | 5 + .../backend/tests/test_llama_cpp_tool_loop.py | 91 +++++++++++++++++++ .../src/components/assistant-ui/reasoning.tsx | 3 +- .../src/features/chat/api/chat-adapter.ts | 19 ++++ .../src/features/chat/api/chat-api.ts | 13 +++ 6 files changed, 171 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index ac9d370b7d..8a7fad9af4 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -7757,6 +7757,15 @@ class LlamaCppBackend: _accumulated_completion_tokens = 0 _accumulated_predicted_ms = 0.0 _accumulated_predicted_n = 0 + # GGUF buffers reasoning; emit server-side timing before answer text. + _reasoning_started_at: Optional[float] = None + _reasoning_summary_emitted = False + + def _reasoning_summary_event(started_at: float) -> dict: + return { + "type": "reasoning_summary", + "duration_ms": round((time.monotonic() - started_at) * 1000.0), + } def _strip_tool_markup( text: str, @@ -7894,6 +7903,9 @@ class LlamaCppBackend: content_buffer = "" # Raw content held during BUFFERING content_accum = "" # All content tokens (for tool parsing) reasoning_accum = "" + # Time each reasoning pass so final answers can replace tool timing. + _reasoning_started_at = None + _reasoning_summary_emitted = False cumulative_display = "" # Cumulative yielded text (with ) in_thinking = False has_content_tokens = False @@ -8068,6 +8080,8 @@ class LlamaCppBackend: # between tool iterations). reasoning = delta.get("reasoning_content", "") if reasoning: + if _reasoning_started_at is None: + _reasoning_started_at = time.monotonic() reasoning_accum += reasoning if detect_state == _S_STREAMING: if not in_thinking: @@ -8083,6 +8097,13 @@ class LlamaCppBackend: # ── Content tokens ── token = delta.get("content", "") if token: + # First answer token ends reasoning. + if ( + _reasoning_started_at is not None + and not _reasoning_summary_emitted + ): + _reasoning_summary_emitted = True + yield _reasoning_summary_event(_reasoning_started_at) has_content_tokens = True content_accum += token @@ -8180,9 +8201,10 @@ class LlamaCppBackend: ), } elif reasoning_accum and not has_content_tokens: - # Reasoning-only response: show reasoning as plain - # text, matching the final streaming pass for - # models that put everything in reasoning. + # Reasoning-only reply: show it as plain text. + if _reasoning_started_at is not None and not _reasoning_summary_emitted: + _reasoning_summary_emitted = True + yield _reasoning_summary_event(_reasoning_started_at) cumulative_display = reasoning_accum if not _suppress_visible_output: yield { @@ -8591,6 +8613,8 @@ class LlamaCppBackend: in_thinking = False has_content_tokens = False reasoning_text = "" + _final_reasoning_started_at: Optional[float] = None + _final_reasoning_summary_emitted = False _metadata_usage = None _metadata_timings = None _metadata_finish_reason = None @@ -8616,6 +8640,12 @@ class LlamaCppBackend: continue if line == "data: [DONE]": if in_thinking: + if ( + _final_reasoning_started_at is not None + and not _final_reasoning_summary_emitted + ): + _final_reasoning_summary_emitted = True + yield _reasoning_summary_event(_final_reasoning_started_at) if has_content_tokens: cumulative += "" yield { @@ -8648,6 +8678,8 @@ class LlamaCppBackend: reasoning = delta.get("reasoning_content", "") if reasoning: + if _final_reasoning_started_at is None: + _final_reasoning_started_at = time.monotonic() reasoning_text += reasoning if not in_thinking: cumulative += "" @@ -8657,6 +8689,12 @@ class LlamaCppBackend: token = delta.get("content", "") if token: + if ( + _final_reasoning_started_at is not None + and not _final_reasoning_summary_emitted + ): + _final_reasoning_summary_emitted = True + yield _reasoning_summary_event(_final_reasoning_started_at) has_content_tokens = True if in_thinking: cumulative += "" diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 53d81961c3..7010fcfabc 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -5101,6 +5101,11 @@ async def openai_chat_completions( _stream_finish = event.get("finish_reason") continue + if event["type"] == "reasoning_summary": + # Forward server-side reasoning timing to the UI. + yield f"data: {json.dumps(event)}\n\n" + continue + # "content" type -- cumulative text. Sanitize the full # cumulative then diff against the last sanitized # snapshot so cross-chunk XML tags are handled correctly. diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py index 687829980b..56e028bd5a 100644 --- a/studio/backend/tests/test_llama_cpp_tool_loop.py +++ b/studio/backend/tests/test_llama_cpp_tool_loop.py @@ -77,6 +77,23 @@ def _tool_names(payload: dict) -> list[str]: ] +def _patch_monotonic(monkeypatch, values: list[float]) -> None: + import core.inference.llama_cpp as llama_cpp_mod + + it = iter(values) + last = values[-1] + + def fake_monotonic() -> float: + nonlocal last + try: + last = next(it) + except StopIteration: + pass + return last + + monkeypatch.setattr(llama_cpp_mod.time, "monotonic", fake_monotonic) + + def _structured_tool_call(tool_name: str, arguments: dict, call_id: str) -> list[str]: return [ _sse( @@ -200,6 +217,80 @@ def test_structured_tool_call_after_visible_preface_is_executed(monkeypatch): assert assistant_messages[-1]["tool_calls"][0]["function"]["name"] == "render_html" +def test_buffered_reasoning_answer_emits_backend_summary(monkeypatch): + stream = [ + _sse({"reasoning_content": "I am thinking."}), + _sse({"reasoning_content": " Still thinking."}), + _sse({"content": "Final answer."}), + _done(), + ] + payloads: list[dict] = [] + backend = _make_backend(monkeypatch, [stream], payloads) + _patch_monotonic(monkeypatch, [100.0, 110.0, 172.0, 172.0]) + + events = list( + backend.generate_chat_completion_with_tools( + messages = [{"role": "user", "content": "answer"}], + tools = [{"type": "function", "function": {"name": "web_search"}}], + max_tool_iterations = 1, + ) + ) + + summary_index = next( + i for i, event in enumerate(events) if event["type"] == "reasoning_summary" + ) + content_index = next(i for i, event in enumerate(events) if event["type"] == "content") + assert summary_index < content_index + assert events[summary_index]["duration_ms"] == 62000 + assert ( + events[content_index]["text"] + == "I am thinking. Still thinking.Final answer." + ) + + +def test_consumed_tool_final_pass_emits_latest_reasoning_summary(monkeypatch): + tool_stream = [ + _sse({"reasoning_content": "Need a render."}), + _sse( + { + "content": '{"name":"render_html","arguments":{"code":"ok"}}' + } + ), + _done(), + ] + final_stream = [ + _sse({"reasoning_content": "Now synthesize."}), + _sse({"content": "Final from tool."}), + _done(), + ] + payloads: list[dict] = [] + backend = _make_backend(monkeypatch, [tool_stream, final_stream], payloads) + _patch_monotonic(monkeypatch, [200.0, 201.0, 203.0, 300.0, 400.0, 405.0, 405.0]) + + def fake_execute_tool(name, arguments, **_kwargs): + return "Rendered HTML canvas: Done." + + monkeypatch.setattr("core.inference.tools.execute_tool", fake_execute_tool) + + events = list( + backend.generate_chat_completion_with_tools( + messages = [{"role": "user", "content": "render then answer"}], + tools = [{"type": "function", "function": {"name": "render_html"}}], + max_tool_iterations = 1, + ) + ) + + summaries = [event for event in events if event["type"] == "reasoning_summary"] + assert [event["duration_ms"] for event in summaries] == [2000, 5000] + final_summary_index = events.index(summaries[-1]) + final_content_index = next( + i + for i, event in enumerate(events) + if event.get("type") == "content" and "Final from tool." in event.get("text", "") + ) + assert final_summary_index < final_content_index + + def test_repeat_render_html_nudge_is_not_user_visible_error(monkeypatch): """A repeated render_html call is an internal no-op, not a visible card.""" diff --git a/studio/frontend/src/components/assistant-ui/reasoning.tsx b/studio/frontend/src/components/assistant-ui/reasoning.tsx index b81c2f2176..c5b53577cb 100644 --- a/studio/frontend/src/components/assistant-ui/reasoning.tsx +++ b/studio/frontend/src/components/assistant-ui/reasoning.tsx @@ -394,7 +394,8 @@ const ReasoningGroupImpl: ReasoningGroupComponent = ({
{isOpen && !isReasoningStreaming && ( diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index a58cdd94d7..5e8fc9cb25 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -2627,6 +2627,17 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { continue; } + // Local GGUF sends server-timed reasoning duration. Guard the type + // so a malformed or proxied chunk (string/null/NaN duration) can + // never turn the label into NaN. + const reasoningMs = ( + chunk as { _reasoningDurationMs?: number } | null | undefined + )?._reasoningDurationMs; + if (typeof reasoningMs === "number" && Number.isFinite(reasoningMs)) { + reasoningDuration = Math.max(0, Math.round(reasoningMs / 1000)); + continue; + } + // Diffusion frame: a transient canvas snapshot. Route it to the transient // store (the in-bubble renderer reads it) and skip it; it has no assistant // text, so it never enters the transcript or the counters below. @@ -3175,6 +3186,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { } const textParts = parseAssistantContent(cumulativeText); + // Fallback when no server-side reasoning_summary arrives. if ( textParts.some((part) => part.type === "reasoning") && !reasoningStartAt @@ -3283,6 +3295,13 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter { finalTokPerSec, ); + // Finalize reasoning-only streams. + if (reasoningStartAt && !reasoningDuration) { + reasoningDuration = Math.max( + 0, + Math.round((Date.now() - reasoningStartAt) / 1000), + ); + } yield { content: [ ...buildAssistantContent(cumulativeText), diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 589fae5fe9..7112a7877c 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -902,6 +902,19 @@ export async function* streamChatCompletions( separatorIndex = buffer.search(/\r?\n\r?\n/); continue; } + // Relay server-side reasoning duration. + if ( + parsed && + typeof parsed === "object" && + "type" in parsed && + parsed.type === "reasoning_summary" + ) { + yield { + _reasoningDurationMs: (parsed as { duration_ms?: number }).duration_ms, + } as unknown as OpenAIChatChunk; + separatorIndex = buffer.search(/\r?\n\r?\n/); + continue; + } yield parsed as OpenAIChatChunk; separatorIndex = buffer.search(/\r?\n\r?\n/); }