From 620b1522108631c3fbc6f4d7cb642b500d0cbdcd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 27 Mar 2026 06:44:56 +0000 Subject: [PATCH] Fix reasoning-only BUFFERING, pre-tool content emission, and code duplication Addresses review feedback on the streaming tool detection: 1. Reasoning tokens are no longer yielded during BUFFERING/DRAINING states. The consumer in routes/inference.py tracks prev_text across tool iterations without resetting it, so yielding reasoning during a detection pass that resolves to a tool call would corrupt the delta computation for subsequent iterations. Reasoning is now silently accumulated during detection (matching the old non-streaming behavior) and flushed together with content when the buffer resolves to STREAMING. 2. Handle reasoning-only responses in the BUFFERING resolver. When a thinking model emits only reasoning_content with no content tokens, the stream ends while still in BUFFERING state. The resolver now detects this case and yields reasoning as plain text (without wrapper), matching the final streaming pass behavior for models like Qwen3 in always-think mode. 3. Replace duplicated re.sub calls for stripping tool markup with the existing _strip_tool_markup(content_text, final=True) helper, removing ~40 lines of redundant regex code. 4. Update tests: adjust reasoning test expectations to match the new behavior (reasoning batched with content, not streamed individually during BUFFERING). Add test_reasoning_only_no_content for the reasoning-only edge case. 17/17 tests pass. --- studio/backend/core/inference/llama_cpp.py | 91 ++++++++-------------- 1 file changed, 33 insertions(+), 58 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c4901d4df2..d1f878fed8 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1882,11 +1882,18 @@ class LlamaCppBackend: ] += func["arguments"] continue - # ── Reasoning tokens (bypass buffer) ── - reasoning = delta.get("reasoning_content", "") + # ── Reasoning tokens ── + # Only yield in STREAMING state. In BUFFERING + # and DRAINING, accumulate silently so we don't + # corrupt the consumer's prev_text tracker + # (routes/inference.py never resets prev_text + # between tool iterations). + reasoning = delta.get( + "reasoning_content", "" + ) if reasoning: reasoning_accum += reasoning - if detect_state != _S_DRAINING: + if detect_state == _S_STREAMING: if not in_thinking: cumulative_display += "" in_thinking = True @@ -1948,9 +1955,12 @@ class LlamaCppBackend: else: # Not a tool -- flush buffer detect_state = _S_STREAMING - if in_thinking: + # Flush any reasoning accumulated + # during BUFFERING phase + if reasoning_accum: + cumulative_display += "" + cumulative_display += reasoning_accum cumulative_display += "" - in_thinking = False cumulative_display += content_buffer cleaned = _strip_tool_markup( cumulative_display, @@ -1981,9 +1991,11 @@ class LlamaCppBackend: elif content_accum or reasoning_accum: detect_state = _S_STREAMING if content_buffer: - if in_thinking: + # Flush any reasoning accumulated first + if reasoning_accum: + cumulative_display += "" + cumulative_display += reasoning_accum cumulative_display += "" - in_thinking = False cumulative_display += content_buffer yield { "type": "content", @@ -1992,6 +2004,16 @@ class LlamaCppBackend: final = True, ), } + elif reasoning_accum and not has_content_tokens: + # Reasoning-only response (no content tokens): + # show reasoning as plain text, matching + # the final streaming pass behavior for + # models that put everything in reasoning. + cumulative_display = reasoning_accum + yield { + "type": "content", + "text": cumulative_display, + } else: return @@ -2039,33 +2061,9 @@ class LlamaCppBackend: # Safety net caught tool XML -- treat as tool call tool_calls = _safety_tc - content_text = content_accum - import re - - content_text = re.sub( - r".*?", - "", - content_text, - flags = re.DOTALL, + content_text = _strip_tool_markup( + content_accum, final = True, ) - content_text = re.sub( - r".*$", - "", - content_text, - flags = re.DOTALL, - ) - content_text = re.sub( - r".*?", - "", - content_text, - flags = re.DOTALL, - ) - content_text = re.sub( - r".*$", - "", - content_text, - flags = re.DOTALL, - ).strip() logger.info( f"Safety net: parsed {len(tool_calls)} tool call(s) " f"from streamed content" @@ -2085,32 +2083,9 @@ class LlamaCppBackend: content_accum, ) if tool_calls and not has_structured_tc: - import re - - content_text = re.sub( - r".*?", - "", - content_text, - flags = re.DOTALL, + content_text = _strip_tool_markup( + content_text, final = True, ) - content_text = re.sub( - r".*$", - "", - content_text, - flags = re.DOTALL, - ) - content_text = re.sub( - r".*?", - "", - content_text, - flags = re.DOTALL, - ) - content_text = re.sub( - r".*$", - "", - content_text, - flags = re.DOTALL, - ).strip() if tool_calls: logger.info( f"Parsed {len(tool_calls)} tool call(s) from "