diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py index bc792c3b99..cdb0fdebff 100644 --- a/studio/backend/core/inference/anthropic_compat.py +++ b/studio/backend/core/inference/anthropic_compat.py @@ -218,6 +218,8 @@ class AnthropicStreamEmitter: def __init__(self) -> None: self.block_index: int = 0 self._text_block_open: bool = False + self._open_tool_call_id: Optional[str] = None + self._open_tool_args_sent: bool = False self._prev_text: str = "" self._usage: dict = {} @@ -263,8 +265,10 @@ class AnthropicStreamEmitter: def finish(self, stop_reason: str = "end_turn") -> list[str]: """Close any open block and emit message_delta + message_stop.""" events = [] - if self._text_block_open: + if self._text_block_open or self._open_tool_call_id is not None: events.append(self._close_block()) + self._open_tool_call_id = None + self._open_tool_args_sent = False events.append( build_anthropic_sse_event( "message_delta", @@ -310,12 +314,26 @@ class AnthropicStreamEmitter: return events def _handle_tool_start(self, event: dict) -> list[str]: + tool_call_id = event.get("tool_call_id", "") + args = event.get("arguments", {}) + if tool_call_id and self._open_tool_call_id == tool_call_id: + return self._tool_arguments_delta(args) + events = [] - # Close current text block if open + # Close current text block if open. if self._text_block_open: events.append(self._close_block()) - # Open a tool_use block + # Defensive: if a replacement/different tool_start arrives while a + # tool_use block is open, close the stale block before starting another. + elif self._open_tool_call_id is not None: + events.append(self._close_block()) + self._open_tool_call_id = None + self._open_tool_args_sent = False + + # Open a tool_use block. self.block_index += 1 + self._open_tool_call_id = tool_call_id + self._open_tool_args_sent = False events.append( build_anthropic_sse_event( "content_block_start", @@ -324,35 +342,43 @@ class AnthropicStreamEmitter: "index": self.block_index, "content_block": { "type": "tool_use", - "id": event.get("tool_call_id", ""), + "id": tool_call_id, "name": event.get("tool_name", ""), "input": {}, }, }, ) ) - # Emit the arguments as input_json_delta - args = event.get("arguments", {}) - if args: - events.append( - build_anthropic_sse_event( - "content_block_delta", - { - "type": "content_block_delta", - "index": self.block_index, - "delta": { - "type": "input_json_delta", - "partial_json": json.dumps(args), - }, - }, - ) - ) + events.extend(self._tool_arguments_delta(args)) return events + def _tool_arguments_delta(self, args: dict) -> list[str]: + if not args: + return [] + if self._open_tool_args_sent: + return [] + self._open_tool_args_sent = True + return [ + build_anthropic_sse_event( + "content_block_delta", + { + "type": "content_block_delta", + "index": self.block_index, + "delta": { + "type": "input_json_delta", + "partial_json": json.dumps(args), + }, + }, + ) + ] + def _handle_tool_end(self, event: dict) -> list[str]: events = [] - # Close the tool_use block - events.append(self._close_block()) + # Close the tool_use block. + if self._open_tool_call_id is not None or self._text_block_open: + events.append(self._close_block()) + self._open_tool_call_id = None + self._open_tool_args_sent = False # Emit custom tool_result event (non-standard, ignored by SDKs) events.append( build_anthropic_sse_event( diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index cce95fc34c..7bcf02dc35 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -52,6 +52,7 @@ from utils.subprocess_compat import ( windows_hidden_subprocess_kwargs as _windows_hidden_subprocess_kwargs, ) from core.inference.tool_call_parser import ( + RENDER_HTML_REPEAT_NUDGE, parse_tool_calls_from_text as _shared_parse_tool_calls_from_text, ) @@ -4616,6 +4617,7 @@ class LlamaCppBackend: # a transient failure are allowed (only block when the previous # identical call succeeded). _tool_call_history: list[tuple[str, bool]] = [] # (key, failed) + _render_html_succeeded = False # ── Re-prompt on plan-without-action ───────────────── # When the model describes what it intends to do (forward-looking @@ -4690,6 +4692,7 @@ class LlamaCppBackend: _iter_timings = None _stream_done = False _last_emitted = "" + provisional_render_html_tool_call_ids = set() stream_timeout = httpx.Timeout( connect = 10, @@ -4799,6 +4802,33 @@ class LlamaCppBackend: tool_calls_acc[idx]["function"][ "arguments" ] += func["arguments"] + current_name = tool_calls_acc[idx][ + "function" + ].get("name", "") + fallback_id = f"call_{idx}" + current_id = tool_calls_acc[idx].get( + "id", fallback_id + ) + already_started = ( + current_id + in provisional_render_html_tool_call_ids + ) + has_real_id = current_id != fallback_id + if ( + current_name == "render_html" + and not _render_html_succeeded + and not already_started + and has_real_id + ): + provisional_render_html_tool_call_ids.add( + current_id + ) + yield { + "type": "tool_start", + "tool_name": "render_html", + "tool_call_id": current_id, + "arguments": {}, + } continue # ── Reasoning tokens ── @@ -4980,13 +5010,25 @@ class LlamaCppBackend: "content": _stripped, } ) + available_tool_names = [ + tool.get("function", {}).get("name") + for tool in tools + if isinstance(tool, dict) + and isinstance(tool.get("function"), dict) + ] + available_tool_names = [ + name for name in available_tool_names if name + ] + tool_hint = ( + " or ".join(available_tool_names) or "an available tool" + ) conversation.append( { "role": "user", "content": ( "STOP. Do NOT write code or explain. " "You MUST call a tool NOW. " - "Call web_search or python immediately." + f"Call {tool_hint} immediately." ), } ) @@ -5158,7 +5200,12 @@ class LlamaCppBackend: arguments = json.loads(raw_args) except (json.JSONDecodeError, ValueError): if auto_heal_tool_calls: - arguments = {"query": raw_args} + heal_key = { + "python": "code", + "terminal": "command", + "render_html": "code", + }.get(tool_name, "query") + arguments = {heal_key: raw_args} else: arguments = {"raw": raw_args} else: @@ -5195,14 +5242,18 @@ class LlamaCppBackend: ) else: status_text = f"Calling: {tool_name}" - yield {"type": "status", "text": status_text} + _repeat_render_html = ( + tool_name == "render_html" and _render_html_succeeded + ) + if not _repeat_render_html: + yield {"type": "status", "text": status_text} - yield { - "type": "tool_start", - "tool_name": tool_name, - "tool_call_id": tc.get("id", ""), - "arguments": arguments, - } + yield { + "type": "tool_start", + "tool_name": tool_name, + "tool_call_id": tc.get("id", ""), + "arguments": arguments, + } # ── Duplicate call detection ────────────── # str(dict) is stable here: arguments always comes from @@ -5210,7 +5261,9 @@ class LlamaCppBackend: # so insertion order is deterministic (Python 3.7+). _tc_key = tool_name + str(arguments) _prev = _tool_call_history[-1] if _tool_call_history else None - if _prev and _prev[0] == _tc_key and not _prev[1]: + if _repeat_render_html: + result = RENDER_HTML_REPEAT_NUDGE + elif _prev and _prev[0] == _tc_key and not _prev[1]: result = ( "You already made this exact call. " "Do not repeat the same tool call. " @@ -5248,12 +5301,13 @@ class LlamaCppBackend: session_id = session_id, ) - yield { - "type": "tool_end", - "tool_name": tool_name, - "tool_call_id": tc.get("id", ""), - "result": result, - } + if not _repeat_render_html: + yield { + "type": "tool_end", + "tool_name": tool_name, + "tool_call_id": tc.get("id", ""), + "result": result, + } # Nudge model to try a different approach on errors _error_prefixes = ( @@ -5269,6 +5323,8 @@ class LlamaCppBackend: _is_error = isinstance(result, str) and result.lstrip().startswith( _error_prefixes ) + if tool_name == "render_html" and not _is_error: + _render_html_succeeded = True _tool_call_history.append((_tc_key, _is_error)) # Strip image sentinel before feeding result to the LLM # (the full result with sentinel is still yielded via diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py index 73bb3d090a..94e9e303ab 100644 --- a/studio/backend/core/inference/safetensors_agentic.py +++ b/studio/backend/core/inference/safetensors_agentic.py @@ -18,6 +18,7 @@ cumulative text and dispatches them via ``core.inference.tools``. """ import json +import re import threading from typing import Callable, Generator, Optional from urllib.parse import urlparse @@ -27,6 +28,7 @@ from loggers import get_logger from core.inference.tool_call_parser import ( BUDGET_EXHAUSTED_NUDGE, DUPLICATE_CALL_NUDGE, + RENDER_HTML_REPEAT_NUDGE, TOOL_ERROR_NUDGE, TOOL_ERROR_PREFIXES, TOOL_XML_SIGNALS, @@ -66,7 +68,34 @@ def _status_for_tool(tool_name: str, arguments: dict) -> str: return f"Calling: {tool_name}" -_CANONICAL_HEAL_ARG = {"python": "code", "terminal": "command"} +_CANONICAL_HEAL_ARG = { + "python": "code", + "terminal": "command", + "render_html": "code", +} + + +_FUNCTION_SIGNAL_RE = re.compile(r"") +_TOOL_CALL_NAME_RE = re.compile(r'"name"\s*:\s*"([\w-]+)"') + + +def _detect_render_html_tool_start(content: str) -> bool: + """Return True when the first drained tool call is clearly render_html.""" + function_match = _FUNCTION_SIGNAL_RE.search(content) + tool_call_index = content.find("") + if not function_match and tool_call_index < 0: + return False + + if function_match and ( + tool_call_index < 0 or function_match.start() < tool_call_index + ): + return function_match.group(1) == "render_html" + + if tool_call_index >= 0: + name_match = _TOOL_CALL_NAME_RE.search(content[tool_call_index:]) + return bool(name_match and name_match.group(1) == "render_html") + + return False def _coerce_arguments(raw_args, *, heal: bool, tool_name: str = "") -> dict: @@ -135,6 +164,7 @@ def run_safetensors_tool_loop( """ conversation = list(messages) tool_call_history: list[tuple[str, bool]] = [] + render_html_succeeded = False final_attempt_done = False allowed_tool_names = { (tool.get("function") or {}).get("name") @@ -161,6 +191,8 @@ def run_safetensors_tool_loop( content_accum = "" cumulative_display = "" last_emitted = "" + provisional_render_html_started = False + provisional_render_html_id = f"call_{next_call_id}" gen = single_turn(conversation) prev_cumulative = "" @@ -179,6 +211,18 @@ def run_safetensors_tool_loop( content_accum += delta if detect_state == _state_draining: + if ( + not render_html_succeeded + and not provisional_render_html_started + and _detect_render_html_tool_start(content_accum) + ): + provisional_render_html_started = True + yield { + "type": "tool_start", + "tool_name": "render_html", + "tool_call_id": provisional_render_html_id, + "arguments": {}, + } continue if detect_state == _state_streaming: @@ -196,6 +240,18 @@ def run_safetensors_tool_loop( yield {"type": "content", "text": cleaned_before} cumulative_display = candidate detect_state = _state_draining + if ( + not render_html_succeeded + and not provisional_render_html_started + and _detect_render_html_tool_start(content_accum) + ): + provisional_render_html_started = True + yield { + "type": "tool_start", + "tool_name": "render_html", + "tool_call_id": provisional_render_html_id, + "arguments": {}, + } continue cumulative_display = candidate cleaned = strip_tool_markup(cumulative_display) @@ -222,6 +278,18 @@ def run_safetensors_tool_loop( if is_match: detect_state = _state_draining + if ( + not render_html_succeeded + and not provisional_render_html_started + and _detect_render_html_tool_start(content_accum) + ): + provisional_render_html_started = True + yield { + "type": "tool_start", + "tool_name": "render_html", + "tool_call_id": provisional_render_html_id, + "arguments": {}, + } elif is_prefix and len(stripped) < _MAX_BUFFER_CHARS: continue else: @@ -282,6 +350,13 @@ def run_safetensors_tool_loop( # literal "" prose is preserved. if content_accum: yield {"type": "content", "text": content_accum} + if provisional_render_html_started: + yield { + "type": "tool_end", + "tool_name": "render_html", + "tool_call_id": provisional_render_html_id, + "result": "Error: render_html tool call could not be parsed.", + } yield {"type": "status", "text": ""} return content_text = strip_tool_markup(content_accum, final = True) @@ -308,16 +383,20 @@ def run_safetensors_tool_loop( tool_name = tool_name, ) - yield {"type": "status", "text": _status_for_tool(tool_name, arguments)} - yield { - "type": "tool_start", - "tool_name": tool_name, - "tool_call_id": tc.get("id", ""), - "arguments": arguments, - } + repeat_render_html = tool_name == "render_html" and render_html_succeeded + if not repeat_render_html: + yield {"type": "status", "text": _status_for_tool(tool_name, arguments)} + yield { + "type": "tool_start", + "tool_name": tool_name, + "tool_call_id": tc.get("id", ""), + "arguments": arguments, + } tc_key = tool_name + str(arguments) - if allowed_tool_names and tool_name not in allowed_tool_names: + if repeat_render_html: + result = RENDER_HTML_REPEAT_NUDGE + elif allowed_tool_names and tool_name not in allowed_tool_names: result = ( f"Error: tool '{tool_name}' is not enabled for this " "request. Use one of the enabled tools or provide a " @@ -345,16 +424,19 @@ def run_safetensors_tool_loop( logger.exception("Tool %s raised: %s", tool_name, exc) result = f"Error: tool raised an exception: {exc}" - yield { - "type": "tool_end", - "tool_name": tool_name, - "tool_call_id": tc.get("id", ""), - "result": result, - } + if not repeat_render_html: + yield { + "type": "tool_end", + "tool_name": tool_name, + "tool_call_id": tc.get("id", ""), + "result": result, + } is_error = isinstance(result, str) and result.lstrip().startswith( TOOL_ERROR_PREFIXES ) + if tool_name == "render_html" and not is_error: + render_html_succeeded = True tool_call_history.append((tc_key, is_error)) # Strip frontend image sentinel from the model's view. diff --git a/studio/backend/core/inference/tool_call_parser.py b/studio/backend/core/inference/tool_call_parser.py index 2f94990623..dacbc19ac0 100644 --- a/studio/backend/core/inference/tool_call_parser.py +++ b/studio/backend/core/inference/tool_call_parser.py @@ -49,6 +49,12 @@ DUPLICATE_CALL_NUDGE = ( "provide your final answer now." ) +RENDER_HTML_REPEAT_NUDGE = ( + "Error: render_html was already called for this response. Do not call " + "render_html again in this response unless the user asks for changes. " + "Provide the final answer now." +) + TOOL_ERROR_NUDGE = ( "\n\nThe tool call encountered an issue. Please try a different " "approach or rephrase your request." @@ -70,6 +76,20 @@ _TC_FUNC_CLOSE_RE = re.compile(r"\s*\s*$") # `issue-number`, `repo-name`); using `\w+` here dropped those keys. _TC_PARAM_START_RE = re.compile(r"\s*") _TC_PARAM_CLOSE_RE = re.compile(r"\s*\s*$") +_PARAM_CLOSE_TAG = "" +_FUNC_CLOSE_TAG = "" + + +def _inside_open_parameter(content: str, pos: int) -> bool: + """Return True when ``pos`` falls inside an unclosed parameter value.""" + last_param_start = -1 + for match in _TC_PARAM_START_RE.finditer(content, 0, pos): + last_param_start = match.start() + if last_param_start < 0: + return False + last_param_close = content.rfind(_PARAM_CLOSE_TAG, 0, pos) + last_func_close = content.rfind(_FUNC_CLOSE_TAG, 0, pos) + return last_param_start > max(last_param_close, last_func_close) def strip_tool_markup(text: str, *, final: bool = False) -> str: @@ -151,7 +171,11 @@ def parse_tool_calls_from_text(content: str, *, id_offset: int = 0) -> list[dict # optional; don't use as body boundary because code # values can contain that literal. if not tool_calls: - func_starts = list(_TC_FUNC_START_RE.finditer(content)) + func_starts = [ + fm + for fm in _TC_FUNC_START_RE.finditer(content) + if not _inside_open_parameter(content, fm.start()) + ] for idx, fm in enumerate(func_starts): func_name = fm.group(1) body_start = fm.end() diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index baf1236456..eecb84ca27 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -514,7 +514,35 @@ TERMINAL_TOOL = { }, } -ALL_TOOLS = [WEB_SEARCH_TOOL, PYTHON_TOOL, TERMINAL_TOOL] +RENDER_HTML_TOOL = { + "type": "function", + "function": { + "name": "render_html", + "description": ( + "Render a self-contained HTML/CSS/JavaScript artifact for the user. " + "Call this at most once per assistant response unless the user " + "explicitly asks for changes in that response. Future user requests " + "for new artifacts may call render_html once. Put the entire document " + "in code, including any CSS in