studio: report the true reasoning duration and fix Stop for thinking models (#6521)
* studio: report the true reasoning duration and fix the Stop button for thinking models For a local GGUF the "Thought for N" label was timed entirely on the client by a brittle edge-detector, so an always-think model (Qwen3 MTP) that buffers its whole reasoning and flushes it in one chunk showed "1 second" instead of the real minute-plus. The client cannot time reasoning it receives atomically, so make the timing backend-authoritative. Backend: generate_chat_completion_with_tools measures wall-clock reasoning and emits a Studio reasoning_summary event (duration_ms) at the moment reasoning ends -- the first answer token, or end-of-stream for a reasoning-only reply -- for both the tool-detection pass and the final-answer pass. Timing resets per tool iteration so the final answer's thinking time wins on the client (which takes the latest reasoning_summary). routes/inference.py forwards the event in the GGUF tool stream. Frontend: parse the reasoning_summary SSE into a _reasoningDurationMs chunk and use it as the authoritative reasoning duration (last write wins), clamped to >= 0 and guarded to a finite number so a malformed or proxied chunk cannot produce a NaN label; the persisted value wins for the final "Thought for N" label, with the previous live timer kept only as a fallback when no metadata arrives. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Daniel Han
GitHub
parent
1ffffc1cf1
commit
6866362da7
6 changed files with 171 additions and 4 deletions
|
|
@ -7757,6 +7757,15 @@ class LlamaCppBackend:
|
|||
_accumulated_completion_tokens = 0
|
||||
_accumulated_predicted_ms = 0.0
|
||||
_accumulated_predicted_n = 0
|
||||
# GGUF buffers reasoning; emit server-side timing before answer text.
|
||||
_reasoning_started_at: Optional[float] = None
|
||||
_reasoning_summary_emitted = False
|
||||
|
||||
def _reasoning_summary_event(started_at: float) -> dict:
|
||||
return {
|
||||
"type": "reasoning_summary",
|
||||
"duration_ms": round((time.monotonic() - started_at) * 1000.0),
|
||||
}
|
||||
|
||||
def _strip_tool_markup(
|
||||
text: str,
|
||||
|
|
@ -7894,6 +7903,9 @@ class LlamaCppBackend:
|
|||
content_buffer = "" # Raw content held during BUFFERING
|
||||
content_accum = "" # All content tokens (for tool parsing)
|
||||
reasoning_accum = ""
|
||||
# Time each reasoning pass so final answers can replace tool timing.
|
||||
_reasoning_started_at = None
|
||||
_reasoning_summary_emitted = False
|
||||
cumulative_display = "" # Cumulative yielded text (with <think>)
|
||||
in_thinking = False
|
||||
has_content_tokens = False
|
||||
|
|
@ -8068,6 +8080,8 @@ class LlamaCppBackend:
|
|||
# between tool iterations).
|
||||
reasoning = delta.get("reasoning_content", "")
|
||||
if reasoning:
|
||||
if _reasoning_started_at is None:
|
||||
_reasoning_started_at = time.monotonic()
|
||||
reasoning_accum += reasoning
|
||||
if detect_state == _S_STREAMING:
|
||||
if not in_thinking:
|
||||
|
|
@ -8083,6 +8097,13 @@ class LlamaCppBackend:
|
|||
# ── Content tokens ──
|
||||
token = delta.get("content", "")
|
||||
if token:
|
||||
# First answer token ends reasoning.
|
||||
if (
|
||||
_reasoning_started_at is not None
|
||||
and not _reasoning_summary_emitted
|
||||
):
|
||||
_reasoning_summary_emitted = True
|
||||
yield _reasoning_summary_event(_reasoning_started_at)
|
||||
has_content_tokens = True
|
||||
content_accum += token
|
||||
|
||||
|
|
@ -8180,9 +8201,10 @@ class LlamaCppBackend:
|
|||
),
|
||||
}
|
||||
elif reasoning_accum and not has_content_tokens:
|
||||
# Reasoning-only response: show reasoning as plain
|
||||
# text, matching the final streaming pass for
|
||||
# models that put everything in reasoning.
|
||||
# Reasoning-only reply: show it as plain text.
|
||||
if _reasoning_started_at is not None and not _reasoning_summary_emitted:
|
||||
_reasoning_summary_emitted = True
|
||||
yield _reasoning_summary_event(_reasoning_started_at)
|
||||
cumulative_display = reasoning_accum
|
||||
if not _suppress_visible_output:
|
||||
yield {
|
||||
|
|
@ -8591,6 +8613,8 @@ class LlamaCppBackend:
|
|||
in_thinking = False
|
||||
has_content_tokens = False
|
||||
reasoning_text = ""
|
||||
_final_reasoning_started_at: Optional[float] = None
|
||||
_final_reasoning_summary_emitted = False
|
||||
_metadata_usage = None
|
||||
_metadata_timings = None
|
||||
_metadata_finish_reason = None
|
||||
|
|
@ -8616,6 +8640,12 @@ class LlamaCppBackend:
|
|||
continue
|
||||
if line == "data: [DONE]":
|
||||
if in_thinking:
|
||||
if (
|
||||
_final_reasoning_started_at is not None
|
||||
and not _final_reasoning_summary_emitted
|
||||
):
|
||||
_final_reasoning_summary_emitted = True
|
||||
yield _reasoning_summary_event(_final_reasoning_started_at)
|
||||
if has_content_tokens:
|
||||
cumulative += "</think>"
|
||||
yield {
|
||||
|
|
@ -8648,6 +8678,8 @@ class LlamaCppBackend:
|
|||
|
||||
reasoning = delta.get("reasoning_content", "")
|
||||
if reasoning:
|
||||
if _final_reasoning_started_at is None:
|
||||
_final_reasoning_started_at = time.monotonic()
|
||||
reasoning_text += reasoning
|
||||
if not in_thinking:
|
||||
cumulative += "<think>"
|
||||
|
|
@ -8657,6 +8689,12 @@ class LlamaCppBackend:
|
|||
|
||||
token = delta.get("content", "")
|
||||
if token:
|
||||
if (
|
||||
_final_reasoning_started_at is not None
|
||||
and not _final_reasoning_summary_emitted
|
||||
):
|
||||
_final_reasoning_summary_emitted = True
|
||||
yield _reasoning_summary_event(_final_reasoning_started_at)
|
||||
has_content_tokens = True
|
||||
if in_thinking:
|
||||
cumulative += "</think>"
|
||||
|
|
|
|||
|
|
@ -5101,6 +5101,11 @@ async def openai_chat_completions(
|
|||
_stream_finish = event.get("finish_reason")
|
||||
continue
|
||||
|
||||
if event["type"] == "reasoning_summary":
|
||||
# Forward server-side reasoning timing to the UI.
|
||||
yield f"data: {json.dumps(event)}\n\n"
|
||||
continue
|
||||
|
||||
# "content" type -- cumulative text. Sanitize the full
|
||||
# cumulative then diff against the last sanitized
|
||||
# snapshot so cross-chunk XML tags are handled correctly.
|
||||
|
|
|
|||
|
|
@ -77,6 +77,23 @@ def _tool_names(payload: dict) -> list[str]:
|
|||
]
|
||||
|
||||
|
||||
def _patch_monotonic(monkeypatch, values: list[float]) -> None:
|
||||
import core.inference.llama_cpp as llama_cpp_mod
|
||||
|
||||
it = iter(values)
|
||||
last = values[-1]
|
||||
|
||||
def fake_monotonic() -> float:
|
||||
nonlocal last
|
||||
try:
|
||||
last = next(it)
|
||||
except StopIteration:
|
||||
pass
|
||||
return last
|
||||
|
||||
monkeypatch.setattr(llama_cpp_mod.time, "monotonic", fake_monotonic)
|
||||
|
||||
|
||||
def _structured_tool_call(tool_name: str, arguments: dict, call_id: str) -> list[str]:
|
||||
return [
|
||||
_sse(
|
||||
|
|
@ -200,6 +217,80 @@ def test_structured_tool_call_after_visible_preface_is_executed(monkeypatch):
|
|||
assert assistant_messages[-1]["tool_calls"][0]["function"]["name"] == "render_html"
|
||||
|
||||
|
||||
def test_buffered_reasoning_answer_emits_backend_summary(monkeypatch):
|
||||
stream = [
|
||||
_sse({"reasoning_content": "I am thinking."}),
|
||||
_sse({"reasoning_content": " Still thinking."}),
|
||||
_sse({"content": "Final answer."}),
|
||||
_done(),
|
||||
]
|
||||
payloads: list[dict] = []
|
||||
backend = _make_backend(monkeypatch, [stream], payloads)
|
||||
_patch_monotonic(monkeypatch, [100.0, 110.0, 172.0, 172.0])
|
||||
|
||||
events = list(
|
||||
backend.generate_chat_completion_with_tools(
|
||||
messages = [{"role": "user", "content": "answer"}],
|
||||
tools = [{"type": "function", "function": {"name": "web_search"}}],
|
||||
max_tool_iterations = 1,
|
||||
)
|
||||
)
|
||||
|
||||
summary_index = next(
|
||||
i for i, event in enumerate(events) if event["type"] == "reasoning_summary"
|
||||
)
|
||||
content_index = next(i for i, event in enumerate(events) if event["type"] == "content")
|
||||
assert summary_index < content_index
|
||||
assert events[summary_index]["duration_ms"] == 62000
|
||||
assert (
|
||||
events[content_index]["text"]
|
||||
== "<think>I am thinking. Still thinking.</think>Final answer."
|
||||
)
|
||||
|
||||
|
||||
def test_consumed_tool_final_pass_emits_latest_reasoning_summary(monkeypatch):
|
||||
tool_stream = [
|
||||
_sse({"reasoning_content": "Need a render."}),
|
||||
_sse(
|
||||
{
|
||||
"content": '<tool_call>{"name":"render_html","arguments":{"code":"<html>ok</html>"}}</tool_call>'
|
||||
}
|
||||
),
|
||||
_done(),
|
||||
]
|
||||
final_stream = [
|
||||
_sse({"reasoning_content": "Now synthesize."}),
|
||||
_sse({"content": "Final from tool."}),
|
||||
_done(),
|
||||
]
|
||||
payloads: list[dict] = []
|
||||
backend = _make_backend(monkeypatch, [tool_stream, final_stream], payloads)
|
||||
_patch_monotonic(monkeypatch, [200.0, 201.0, 203.0, 300.0, 400.0, 405.0, 405.0])
|
||||
|
||||
def fake_execute_tool(name, arguments, **_kwargs):
|
||||
return "Rendered HTML canvas: Done."
|
||||
|
||||
monkeypatch.setattr("core.inference.tools.execute_tool", fake_execute_tool)
|
||||
|
||||
events = list(
|
||||
backend.generate_chat_completion_with_tools(
|
||||
messages = [{"role": "user", "content": "render then answer"}],
|
||||
tools = [{"type": "function", "function": {"name": "render_html"}}],
|
||||
max_tool_iterations = 1,
|
||||
)
|
||||
)
|
||||
|
||||
summaries = [event for event in events if event["type"] == "reasoning_summary"]
|
||||
assert [event["duration_ms"] for event in summaries] == [2000, 5000]
|
||||
final_summary_index = events.index(summaries[-1])
|
||||
final_content_index = next(
|
||||
i
|
||||
for i, event in enumerate(events)
|
||||
if event.get("type") == "content" and "Final from tool." in event.get("text", "")
|
||||
)
|
||||
assert final_summary_index < final_content_index
|
||||
|
||||
|
||||
def test_repeat_render_html_nudge_is_not_user_visible_error(monkeypatch):
|
||||
"""A repeated render_html call is an internal no-op, not a visible card."""
|
||||
|
||||
|
|
|
|||
|
|
@ -394,7 +394,8 @@ const ReasoningGroupImpl: ReasoningGroupComponent = ({
|
|||
<ReasoningTrigger
|
||||
className="min-w-0 flex-1"
|
||||
active={isReasoningStreaming}
|
||||
duration={duration || persistedDuration}
|
||||
// Prefer server timing when available.
|
||||
duration={persistedDuration || duration}
|
||||
/>
|
||||
<div className="flex w-16 shrink-0 justify-end">
|
||||
{isOpen && !isReasoningStreaming && (
|
||||
|
|
|
|||
|
|
@ -2627,6 +2627,17 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
continue;
|
||||
}
|
||||
|
||||
// Local GGUF sends server-timed reasoning duration. Guard the type
|
||||
// so a malformed or proxied chunk (string/null/NaN duration) can
|
||||
// never turn the label into NaN.
|
||||
const reasoningMs = (
|
||||
chunk as { _reasoningDurationMs?: number } | null | undefined
|
||||
)?._reasoningDurationMs;
|
||||
if (typeof reasoningMs === "number" && Number.isFinite(reasoningMs)) {
|
||||
reasoningDuration = Math.max(0, Math.round(reasoningMs / 1000));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Diffusion frame: a transient canvas snapshot. Route it to the transient
|
||||
// store (the in-bubble renderer reads it) and skip it; it has no assistant
|
||||
// text, so it never enters the transcript or the counters below.
|
||||
|
|
@ -3175,6 +3186,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
}
|
||||
const textParts = parseAssistantContent(cumulativeText);
|
||||
|
||||
// Fallback when no server-side reasoning_summary arrives.
|
||||
if (
|
||||
textParts.some((part) => part.type === "reasoning") &&
|
||||
!reasoningStartAt
|
||||
|
|
@ -3283,6 +3295,13 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
finalTokPerSec,
|
||||
);
|
||||
|
||||
// Finalize reasoning-only streams.
|
||||
if (reasoningStartAt && !reasoningDuration) {
|
||||
reasoningDuration = Math.max(
|
||||
0,
|
||||
Math.round((Date.now() - reasoningStartAt) / 1000),
|
||||
);
|
||||
}
|
||||
yield {
|
||||
content: [
|
||||
...buildAssistantContent(cumulativeText),
|
||||
|
|
|
|||
|
|
@ -902,6 +902,19 @@ export async function* streamChatCompletions(
|
|||
separatorIndex = buffer.search(/\r?\n\r?\n/);
|
||||
continue;
|
||||
}
|
||||
// Relay server-side reasoning duration.
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed === "object" &&
|
||||
"type" in parsed &&
|
||||
parsed.type === "reasoning_summary"
|
||||
) {
|
||||
yield {
|
||||
_reasoningDurationMs: (parsed as { duration_ms?: number }).duration_ms,
|
||||
} as unknown as OpenAIChatChunk;
|
||||
separatorIndex = buffer.search(/\r?\n\r?\n/);
|
||||
continue;
|
||||
}
|
||||
yield parsed as OpenAIChatChunk;
|
||||
separatorIndex = buffer.search(/\r?\n\r?\n/);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue