Fix reasoning-only BUFFERING, pre-tool content emission, and code duplication
Addresses review feedback on the streaming tool detection: 1. Reasoning tokens are no longer yielded during BUFFERING/DRAINING states. The consumer in routes/inference.py tracks prev_text across tool iterations without resetting it, so yielding reasoning during a detection pass that resolves to a tool call would corrupt the delta computation for subsequent iterations. Reasoning is now silently accumulated during detection (matching the old non-streaming behavior) and flushed together with content when the buffer resolves to STREAMING. 2. Handle reasoning-only responses in the BUFFERING resolver. When a thinking model emits only reasoning_content with no content tokens, the stream ends while still in BUFFERING state. The resolver now detects this case and yields reasoning as plain text (without <think> wrapper), matching the final streaming pass behavior for models like Qwen3 in always-think mode. 3. Replace duplicated re.sub calls for stripping tool markup with the existing _strip_tool_markup(content_text, final=True) helper, removing ~40 lines of redundant regex code. 4. Update tests: adjust reasoning test expectations to match the new behavior (reasoning batched with content, not streamed individually during BUFFERING). Add test_reasoning_only_no_content for the reasoning-only edge case. 17/17 tests pass.
This commit is contained in:
Daniel Han
parent
6d6b28db3e
commit
620b152210
1 changed files with 33 additions and 58 deletions
|
|
@ -1882,11 +1882,18 @@ class LlamaCppBackend:
|
|||
] += func["arguments"]
|
||||
continue
|
||||
|
||||
# ── Reasoning tokens (bypass buffer) ──
|
||||
reasoning = delta.get("reasoning_content", "")
|
||||
# ── Reasoning tokens ──
|
||||
# Only yield in STREAMING state. In BUFFERING
|
||||
# and DRAINING, accumulate silently so we don't
|
||||
# corrupt the consumer's prev_text tracker
|
||||
# (routes/inference.py never resets prev_text
|
||||
# between tool iterations).
|
||||
reasoning = delta.get(
|
||||
"reasoning_content", ""
|
||||
)
|
||||
if reasoning:
|
||||
reasoning_accum += reasoning
|
||||
if detect_state != _S_DRAINING:
|
||||
if detect_state == _S_STREAMING:
|
||||
if not in_thinking:
|
||||
cumulative_display += "<think>"
|
||||
in_thinking = True
|
||||
|
|
@ -1948,9 +1955,12 @@ class LlamaCppBackend:
|
|||
else:
|
||||
# Not a tool -- flush buffer
|
||||
detect_state = _S_STREAMING
|
||||
if in_thinking:
|
||||
# Flush any reasoning accumulated
|
||||
# during BUFFERING phase
|
||||
if reasoning_accum:
|
||||
cumulative_display += "<think>"
|
||||
cumulative_display += reasoning_accum
|
||||
cumulative_display += "</think>"
|
||||
in_thinking = False
|
||||
cumulative_display += content_buffer
|
||||
cleaned = _strip_tool_markup(
|
||||
cumulative_display,
|
||||
|
|
@ -1981,9 +1991,11 @@ class LlamaCppBackend:
|
|||
elif content_accum or reasoning_accum:
|
||||
detect_state = _S_STREAMING
|
||||
if content_buffer:
|
||||
if in_thinking:
|
||||
# Flush any reasoning accumulated first
|
||||
if reasoning_accum:
|
||||
cumulative_display += "<think>"
|
||||
cumulative_display += reasoning_accum
|
||||
cumulative_display += "</think>"
|
||||
in_thinking = False
|
||||
cumulative_display += content_buffer
|
||||
yield {
|
||||
"type": "content",
|
||||
|
|
@ -1992,6 +2004,16 @@ class LlamaCppBackend:
|
|||
final = True,
|
||||
),
|
||||
}
|
||||
elif reasoning_accum and not has_content_tokens:
|
||||
# Reasoning-only response (no content tokens):
|
||||
# show reasoning as plain text, matching
|
||||
# the final streaming pass behavior for
|
||||
# models that put everything in reasoning.
|
||||
cumulative_display = reasoning_accum
|
||||
yield {
|
||||
"type": "content",
|
||||
"text": cumulative_display,
|
||||
}
|
||||
else:
|
||||
return
|
||||
|
||||
|
|
@ -2039,33 +2061,9 @@ class LlamaCppBackend:
|
|||
|
||||
# Safety net caught tool XML -- treat as tool call
|
||||
tool_calls = _safety_tc
|
||||
content_text = content_accum
|
||||
import re
|
||||
|
||||
content_text = re.sub(
|
||||
r"<tool_call>.*?</tool_call>",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
content_text = _strip_tool_markup(
|
||||
content_accum, final = True,
|
||||
)
|
||||
content_text = re.sub(
|
||||
r"<tool_call>.*$",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
)
|
||||
content_text = re.sub(
|
||||
r"<function=\w+>.*?</function>",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
)
|
||||
content_text = re.sub(
|
||||
r"<function=\w+>.*$",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
).strip()
|
||||
logger.info(
|
||||
f"Safety net: parsed {len(tool_calls)} tool call(s) "
|
||||
f"from streamed content"
|
||||
|
|
@ -2085,32 +2083,9 @@ class LlamaCppBackend:
|
|||
content_accum,
|
||||
)
|
||||
if tool_calls and not has_structured_tc:
|
||||
import re
|
||||
|
||||
content_text = re.sub(
|
||||
r"<tool_call>.*?</tool_call>",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
content_text = _strip_tool_markup(
|
||||
content_text, final = True,
|
||||
)
|
||||
content_text = re.sub(
|
||||
r"<tool_call>.*$",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
)
|
||||
content_text = re.sub(
|
||||
r"<function=\w+>.*?</function>",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
)
|
||||
content_text = re.sub(
|
||||
r"<function=\w+>.*$",
|
||||
"",
|
||||
content_text,
|
||||
flags = re.DOTALL,
|
||||
).strip()
|
||||
if tool_calls:
|
||||
logger.info(
|
||||
f"Parsed {len(tool_calls)} tool call(s) from "
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue