diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 563a6732a1..0af37e627f 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -2281,8 +2281,13 @@ class InferenceBackend: except Exception as e: logger.warning(f"Could not fully reset model state for {model_name}: {e}") - def reset_generation_state(self): - """Reset any cached generation state to prevent hanging after errors""" + def reset_generation_state(self, caller_cancel_event = None): + """Reset any cached generation state to prevent hanging after errors + + ``caller_cancel_event`` is accepted for signature parity with the + orchestrator, which uses it to drop a reset from a request that never + started. Nothing here cancels a live generation, so it is unused. + """ try: # Clear cached state for ALL loaded models for model_name in self.models.keys(): diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.py index 1a9ae04b0e..db9a5d8ce4 100644 --- a/studio/backend/core/inference/llama_admission.py +++ b/studio/backend/core/inference/llama_admission.py @@ -214,7 +214,7 @@ class _Waiter: class LlamaAdmissionLease: - __slots__ = ("_queue", "_slot", "_released", "_release_lock") + __slots__ = ("_queue", "_slot", "_released", "_release_lock", "_parked") def __init__( self, @@ -225,20 +225,88 @@ class LlamaAdmissionLease: self._slot = slot self._released = False self._release_lock = threading.Lock() + self._parked = False @property def slot(self) -> Optional[int]: """Pool slot this lease holds, or None when admission is disabled.""" return self._slot + def park(self) -> None: + """Hand the slot back while this holder waits on something off the GPU. + + A run stopped on a tool approval prompt is not decoding, so holding its + slot would let unanswered prompts fill the pool while llama-server idles. + The lease itself stays valid: releasing it after a park is still correct. + """ + queue = self._queue + slot = None + with self._release_lock: + if queue is None or self._released or self._parked: + return + self._parked = True + slot, self._slot = self._slot, None + queue.park(slot) + + def unpark(self) -> None: + """Drop the parked state without reclaiming a slot. + + For a holder that is tearing down: it will not decode again. Resuming + holders must use ``unpark_async``, which waits for a slot instead of + going back to llama-server past the admission limit. + """ + with self._release_lock: + if not self._parked: + return + self._parked = False + if self._queue is not None: + self._queue.unpark() + + async def unpark_async( + self, + *, + cancel_event = None, + poll_s: float = 0.02, + ) -> None: + """Take a slot back, waiting until the pool has room. + + ``park`` gave the slot to a waiter, so by the time the user answers the + prompt someone else may be decoding in it. Resuming regardless put two + holders on a one-slot server. Gives up if the caller is cancelled, since + the holder is then leaving anyway and must not be stuck here. + """ + queue = self._queue + if queue is None or not self._parked: + return + slot = await queue.acquire_parked_slot(cancel_event = cancel_event, poll_s = poll_s) + stranded = None + with self._release_lock: + # release() may have run during the wait; it clears the flag and does + # the unpark itself, so only the caller that clears it here repeats one. + parked, self._parked = self._parked, False + if self._released: + # Released while waiting: this lease will never hand the slot + # back, so return it here rather than strand it for good. + stranded = slot + else: + self._slot = slot + if parked: + queue.unpark() + if stranded is not None: + queue.release(stranded) + def release(self) -> None: queue = None + parked = False with self._release_lock: if self._released: return self._released = True queue = self._queue + parked, self._parked = self._parked, False if queue is not None: + if parked: + queue.unpark() queue.release(self._slot) async def __aenter__(self) -> "LlamaAdmissionLease": @@ -338,7 +406,18 @@ class LlamaAdmissionQueue: set to 0. See ``LlamaAdmissionConfig.queue_limit``. """ - __slots__ = ("key", "_lock", "_capacity", "_free", "_in_use", "_held", "_waiters") + __slots__ = ( + "key", + "_lock", + "_capacity", + "_free", + "_in_use", + "_held", + "_waiters", + "_parked", + "_unpark_tickets", + "_unpark_seq", + ) def __init__(self, key: str): self.key = key @@ -351,6 +430,13 @@ class LlamaAdmissionQueue: self._in_use = 0 self._held = 0 self._waiters: Deque[_Waiter] = deque() + # Holders parked on a tool approval prompt. They hold no slot, so this only + # keeps the queue off the idle-eviction list while they are away. + self._parked = 0 + # FIFO tickets for holders resuming from a park (see acquire_parked_slot). A + # bare count deadlocked: every approved holder blocked every other one. + self._unpark_tickets: Deque[int] = deque() + self._unpark_seq = 0 def _resize_pool_locked(self, capacity: int) -> None: # Slots past a shrunk capacity retire when their holder releases them. @@ -359,13 +445,15 @@ class LlamaAdmissionQueue: self._capacity = capacity self._free = [slot for slot in range(capacity) if not self._in_use >> slot & 1] - def _can_admit_locked(self) -> bool: + def _can_admit_locked(self, reserved: int) -> bool: # Slots still held above a shrunk capacity keep occupying the backend, so # count every held slot against the ceiling, not just the ids below it. - return bool(self._free) and self._held < self._capacity + # ``reserved`` holds slots back for approved holders waiting to resume; + # without it a stream of new arrivals took the next slot, forever. + return bool(self._free) and (self._held + reserved) < self._capacity - def _take_slot_locked(self) -> Optional[int]: - if not self._can_admit_locked(): + def _take_slot_locked(self, reserved: int) -> Optional[int]: + if not self._can_admit_locked(reserved): return None slot = self._free.pop() self._in_use |= 1 << slot @@ -386,7 +474,7 @@ class LlamaAdmissionQueue: self._resize_pool_locked(capacity) self._grant_waiters_locked() if not self._waiters: - slot = self._take_slot_locked() + slot = self._take_slot_locked(len(self._unpark_tickets)) if slot is not None: # No snapshot here: callers read it through snapshot_now(), # which re-reads the queue, so building one per admitted @@ -425,6 +513,58 @@ class LlamaAdmissionQueue: self._release_slot_locked(slot) self._grant_waiters_locked() + def park(self, slot: Optional[int]) -> None: + """Return a parked holder's slot to the pool. See ``LlamaAdmissionLease.park``.""" + with self._lock: + self._parked += 1 + self._release_slot_locked(slot) + self._grant_waiters_locked() + + def unpark(self) -> None: + with self._lock: + if self._parked > 0: + self._parked -= 1 + + async def acquire_parked_slot( + self, + *, + cancel_event = None, + poll_s: float = 0.02, + ) -> Optional[int]: + """Wait for a slot for a holder resuming from a park, None if cancelled. + + Ordered by ticket rather than counted, so approvals resume in the order + they came back: counting them made every approved holder block every + other one, and with nothing decoding that never resolved. + """ + with self._lock: + self._unpark_seq += 1 + ticket = self._unpark_seq + self._unpark_tickets.append(ticket) + try: + while True: + with self._lock: + ahead = 0 + for queued in self._unpark_tickets: + if queued == ticket: + break + ahead += 1 + # Only the approvals ahead of this one hold slots back from it. + slot = self._take_slot_locked(ahead) + if slot is not None: + return slot + if cancel_event is not None and cancel_event.is_set(): + return None + await asyncio.sleep(poll_s) + finally: + with self._lock: + try: + self._unpark_tickets.remove(ticket) + except ValueError: + pass + # This ticket was holding a slot back from the wait line. + self._grant_waiters_locked() + def cancel(self, waiter: _Waiter) -> None: lease_to_release = None with self._lock: @@ -455,15 +595,17 @@ class LlamaAdmissionQueue: def is_idle(self) -> bool: with self._lock: self._prune_waiters_locked() - return self._in_use == 0 and not self._waiters + # A parked holder owns no slot but is coming back to this queue, so + # evicting it here would resume it against a fresh 1-slot pool. + return self._in_use == 0 and not self._waiters and not self._parked def _grant_waiters_locked(self) -> None: # Dead waiters are skipped as they are popped, so no prune is needed here. - while self._waiters and self._can_admit_locked(): + while self._waiters and self._can_admit_locked(len(self._unpark_tickets)): waiter = self._waiters.popleft() if waiter.cancelled or waiter.future.done(): continue - slot = self._take_slot_locked() + slot = self._take_slot_locked(len(self._unpark_tickets)) lease = LlamaAdmissionLease(self, slot) waiter.granted_lease = lease try: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c83d3696a8..a23501a6eb 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -98,6 +98,7 @@ from core.inference.tool_call_parser import ( from core.inference.tool_loop_controller import ( ToolLoopController, append_deferred_nudges, + awaiting_approval_status, tool_event_provenance, ) from state.tool_approvals import ( @@ -6551,6 +6552,25 @@ class LlamaCppBackend: binary = self._find_llama_server_binary() is_vulkan_backend = self._is_vulkan_backend(binary) + # Without --kv-unified an explicit --parallel N splits -c into windows of -c/N, so on a + # build lacking the flag the default of 4 would quarter every context window for a + # feature it cannot serve: fall back to one slot. Ahead of the KV estimates so the + # fit matches what launches. + if ( + n_parallel > 1 + and binary + and not self.probe_server_capabilities(binary).get("supports_kv_unified") + ): + logger.warning( + "llama-server at %s has no --kv-unified, so %d parallel slots would " + "split the context window %d ways. Using 1 slot instead; update " + "llama.cpp to run chats in parallel.", + binary, + n_parallel, + n_parallel, + ) + n_parallel = 1 + # ── Vulkan-ordinal preflight (BEFORE the Phase 1 kill) ──────── # An explicit Vulkan pin the ggml probe never enumerated cannot be honored. # Validate it ABOVE the kill so an invalid selection leaves the live model @@ -11101,6 +11121,7 @@ class LlamaCppBackend: from core.inference.tools import ( build_rag_autoinject, execute_tool, + has_text_only_provisional_card, is_always_safe_tool, is_high_risk_tool_call, ) @@ -11527,6 +11548,9 @@ class LlamaCppBackend: permission_mode == "auto" and is_always_safe_tool(current_name) ) + # A text-preview card still streams while gated; + # hiding it blanks the chat. + and not has_text_only_provisional_card(current_name) ) # Keep small-argument tools on the normal path. _args_len = len( @@ -11628,20 +11652,27 @@ class LlamaCppBackend: # TEXT call to a provisional card. Gated on an enabled-name # sniff + size floor so prose/small calls spawn no pane; id # matches the first call so the final tool_start reconciles. - if ( - not has_structured_tc - and not _confirm_gated_iteration - and _text_args_call_start >= 0 - ): + if not has_structured_tc and _text_args_call_start >= 0: if not _text_args_id: _call_text = content_accum[_text_args_call_start:] _sniffed = _sniff_text_tool_name( _call_text, _enabled_tool_names ) - if _sniffed and ( - _sniffed == "render_html" - or len(_call_text) - >= _PROVISIONAL_ARGS_MIN_CHARS + # Structured-path rule: gated calls + # stream only from a text-preview card. + if ( + _sniffed + and not ( + _confirm_gated_iteration + and not has_text_only_provisional_card( + _sniffed + ) + ) + and ( + _sniffed == "render_html" + or len(_call_text) + >= _PROVISIONAL_ARGS_MIN_CHARS + ) ): _text_args_id = "call_0" _text_args_name = _sniffed @@ -12230,18 +12261,31 @@ class LlamaCppBackend: start_event["awaiting_confirmation"] = needs_confirm try: - yield {"type": "status", "text": decision.status_text} + # Gated calls are not running yet; a "Running ..." badge + # counting up while it waits on a human reads as a hang. + yield { + "type": "status", + "text": ( + awaiting_approval_status(decision.tool_name) + if needs_confirm + else decision.status_text + ), + } yield start_event - if ( - decision_slot is not None - and wait_tool_decision( + _decision = ( + wait_tool_decision( decision_slot, approval_id, cancel_event = cancel_event, ) - == "deny" - ): + if decision_slot is not None + else None + ) + if _decision is not None and _decision != "deny": + # Approved: now it really is running. + yield {"type": "status", "text": decision.status_text} + if _decision == "deny": decision_slot = None resolved_provisional_tool_call_ids.add(decision.tool_call_id) yield { @@ -12809,10 +12853,15 @@ class LlamaCppBackend: min_p: float = 0.0, max_new_tokens: int = 2048, repetition_penalty: float = 1.1, + cancel_event: Optional[threading.Event] = None, ) -> tuple: """ Generate TTS audio via llama-server /completion + codec decode. Returns (wav_bytes, sample_rate). + + ``cancel_event`` lets a Stop or a forced model swap end the request: the + decode is one blocking POST, so a watcher closes the client out from under + it rather than polling. Raises RuntimeError once cancelled. """ if audio_type not in self._TTS_PROMPTS: raise RuntimeError(f"GGUF TTS does not support '{audio_type}' codec.") @@ -12834,15 +12883,47 @@ class LlamaCppBackend: if need_ids: payload["n_probs"] = 1 + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") + with httpx.Client( timeout = httpx.Timeout(300, connect = 10), headers = self._auth_headers, trust_env = False, ) as client: - resp = client.post(f"{self.base_url}/completion", json = payload) + finished = threading.Event() + watcher: Optional[threading.Thread] = None + if cancel_event is not None: + + def _close_when_cancelled() -> None: + while not finished.wait(0.05): + if cancel_event.is_set(): + # Closing mid-request makes the blocking post raise + # httpx.RequestError, the only way out of it. + with contextlib.suppress(Exception): + client.close() + return + + watcher = threading.Thread(target = _close_when_cancelled, daemon = True) + watcher.start() + try: + resp = client.post(f"{self.base_url}/completion", json = payload) + except httpx.RequestError: + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") from None + raise + finally: + finished.set() + if watcher is not None: + watcher.join(timeout = 0.5) if resp.status_code != 200: raise RuntimeError(f"llama-server returned {resp.status_code}: {resp.text}") + # The codec decode below is GPU work with no interruption point, so check here: + # cancelling after this only wastes the decode it cannot stop. + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") + data = resp.json() token_ids = ( [p["id"] for p in data.get("completion_probabilities", []) if "id" in p] diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py index d19c67a01a..2b300a32b1 100644 --- a/studio/backend/core/inference/mlx_inference.py +++ b/studio/backend/core/inference/mlx_inference.py @@ -1189,7 +1189,8 @@ class MLXInferenceBackend: **gen_kwargs, ) - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): + # caller_cancel_event: signature parity with the orchestrator; unused here. import mlx.core as mx import gc diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py index 616384386d..4699148a08 100644 --- a/studio/backend/core/inference/orchestrator.py +++ b/studio/backend/core/inference/orchestrator.py @@ -104,6 +104,14 @@ class InferenceOrchestrator: # so a generate queued behind the cancelled one is skipped, not run. self._drain_event: Any = None self._gen_lock = threading.Lock() # Serializes generation + # Cancel event of the request holding _gen_lock: lets a Stop tell whether it owns the + # running generation or is queued behind it (the worker's event is shared). + self._active_cancel_events: list = [] + self._executing_cancel_events: list = [] + self._active_cancel_lock = threading.Lock() + # Held across claim + _send_cmd so claim order matches the subprocess dequeue order, + # which _owns_worker relies on. + self._send_order_lock = threading.Lock() # Set during a switch so a generation winning the _gen_lock handoff bails # instead of starting on the outgoing model. self._unload_pending = False @@ -112,6 +120,13 @@ class InferenceOrchestrator: # bypass _gen_lock, send commands directly, read from per-request # mailboxes routed by a dispatcher thread on request_id. self._mailboxes: dict[str, queue.Queue] = {} + # request_id -> cancel event, so the dispatcher can move worker ownership as it routes. + # Consumers read their mailbox whenever they get to it, so only the dispatcher sees + # responses in the order the worker produced them. + self._request_cancel_events: dict[str, object] = {} + # Mailboxes for the _gen_lock generations. Kept apart from _mailboxes because that map + # means "compare requests are in flight" to the unload and distributed paths. + self._direct_mailboxes: dict[str, queue.Queue] = {} self._mailbox_lock = threading.Lock() self._dispatcher_thread: Optional[threading.Thread] = None self._dispatcher_stop = threading.Event() @@ -321,9 +336,27 @@ class InferenceOrchestrator: self._resp_queue = None self._cancel_event = None self._drain_event = None + self._reset_worker_scoped_state() logger.info("Inference subprocess shut down") return True + def _reset_worker_scoped_state(self) -> None: + """Drop bookkeeping that only means anything for the worker that just died. + + Ownership is scoped by cancel-event identity alone, so a consumer still blocked + on its mailbox when the process was replaced stayed recorded as the executor. A + generation on the fresh worker then failed _owns_worker and could not be stopped. + Mailboxes go too: nothing will ever route to them, and a stale one reads as + compare activity to the unload path. + """ + with self._active_cancel_lock: + self._active_cancel_events.clear() + self._executing_cancel_events.clear() + with self._mailbox_lock: + self._mailboxes.clear() + self._direct_mailboxes.clear() + self._request_cancel_events.clear() + def _cleanup(self): """atexit handler.""" self._shutdown_subprocess(timeout = 5.0) @@ -463,6 +496,74 @@ class InferenceOrchestrator: except (EOFError, OSError, ValueError): return events + def _direct_reader(self, request_id: str): + """Response reader for a _gen_lock generation, safe once compare exists. + + The dispatcher and this reader would otherwise both consume _resp_queue. A + dispatcher started mid-stream took our responses and dropped them as + unaddressed (truncating or hanging the chat), and this reader, already blocked + on the queue, could take a compare request's response before that dispatcher + saw it. Registering a mailbox fixes the first; handing foreign responses to + their own mailbox fixes the second. + + Returns (read_one, drain, release). + """ + mailbox: queue.Queue = queue.Queue() + with self._mailbox_lock: + self._direct_mailboxes[request_id] = mailbox + + def read_one(timeout: float = 1.0): + try: + return mailbox.get_nowait() + except queue.Empty: + pass + thread = self._dispatcher_thread + if thread is not None and thread.is_alive(): + # It owns the queue now, and it routes to us. + try: + return mailbox.get(timeout = timeout) + except queue.Empty: + return None + resp = self._read_resp(timeout = timeout) + if resp is None: + return None + rid = resp.get("request_id") + if rid and rid != request_id: + with self._mailbox_lock: + other = self._mailboxes.get(rid) or self._direct_mailboxes.get(rid) + owner = self._request_cancel_events.get(rid) + if other is not None: + # We beat the dispatcher to this response, so make its ownership move here + # too. The compare consumer opts out of marking, so nothing else promotes + # or retires that request: skipping it left this one recorded as the + # executor, ignoring its Stop and letting a late reset cancel it. + if owner is not None: + if resp.get("type", "") in ("gen_done", "gen_error"): + self._release_worker(owner) + else: + self._mark_worker_started(owner) + other.put(resp) + return None + return resp + + def drain(timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + resp = read_one(timeout = min(0.5, deadline - time.monotonic())) + if resp is None: + if not self._ensure_subprocess_alive(): + return + continue + if resp.get("type", "") in ("gen_done", "gen_error"): + return + logger.warning("Timed out waiting for gen_done after cancel") + + def release() -> None: + with self._mailbox_lock: + self._direct_mailboxes.pop(request_id, None) + + return read_one, drain, release + def _drain_until_gen_done(self, timeout: float = 5.0) -> None: """Consume resp_queue events until gen_done/gen_error, discarding them. @@ -542,6 +643,7 @@ class InferenceOrchestrator: cancel_event = None, stats_holder: Optional[dict] = None, read_timeout: float = 30.0, + mark_started: bool = True, ) -> Generator[str, None, None]: """Yield tokens from a response stream until gen_done/gen_error. @@ -578,6 +680,11 @@ class InferenceOrchestrator: rtype = resp.get("type", "") if rtype == "status": continue + # The worker is answering THIS request, so it is the one executing: only now may its + # cancel event speak for the shared worker one. The dispatched path opts out: its + # dispatcher already did this in worker order, which a mailbox read can lag behind. + if mark_started: + self._mark_worker_started(cancel_event) # Subprocess-level error (no request_id); request-scoped failures # arrive as gen_error below. if rtype == "error" and not resp.get("request_id"): @@ -587,7 +694,13 @@ class InferenceOrchestrator: if rtype == "token": # Cancel from route (e.g. SSE connection closed). if cancel_event is not None and cancel_event.is_set(): - self._cancel_generation() + # Same rule as reset_generation_state: the shared worker event may only be set by + # the generation the worker is running. A dispatched request can still be draining + # stale mailbox tokens after the dispatcher retired it, and signalling from here + # would end the next one instead. Tearing this stream down is always safe, so the + # local drain happens either way. + if self._owns_worker(cancel_event): + self._cancel_generation() drain_on_cancel() return yield resp.get("text", "") @@ -681,8 +794,17 @@ class InferenceOrchestrator: # Route to mailbox if a matching request_id exists if rid: with self._mailbox_lock: - mbox = self._mailboxes.get(rid) + mbox = self._mailboxes.get(rid) or self._direct_mailboxes.get(rid) + owner = self._request_cancel_events.get(rid) if mbox is not None: + # Worker order, not consumer order: retire a request the moment its last response + # is routed. Waiting for the consumer's finally left it owning the worker after + # the worker moved on, so a late Stop for it cancelled whichever request started next. + if owner is not None: + if rtype in ("gen_done", "gen_error"): + self._release_worker(owner) + else: + self._mark_worker_started(owner) mbox.put(resp) continue @@ -798,6 +920,8 @@ class InferenceOrchestrator: ) if not unloading: self._mailboxes[request_id] = mailbox + if cancel_event is not None: + self._request_cancel_events[request_id] = cancel_event # When bailing without a mailbox, note whether any OTHER compare request still # routes through the dispatcher; if none and this call started it, stop it below. orphaned_dispatcher = unloading and not dispatcher_preexisting and not self._mailboxes @@ -813,11 +937,19 @@ class InferenceOrchestrator: yield GenStreamError("Error: model is being unloaded", public = True) return + # Claim before sending, like the locked path: dispatched runs are concurrent by design, + # so without this a Stop on one saw no owner and reset the worker, ending its siblings. + # Claim and enqueue under one lock, or two dispatcher threads interleave and claim order + # stops matching the subprocess's command order, which _owns_worker reads. try: - self._send_cmd(cmd) + with self._send_order_lock: + self._claim_worker(cancel_event) + self._send_cmd(cmd) except RuntimeError as exc: + self._release_worker(cancel_event) with self._mailbox_lock: self._mailboxes.pop(request_id, None) + self._request_cancel_events.pop(request_id, None) yield GenStreamError(f"Error: {exc}") return @@ -836,10 +968,15 @@ class InferenceOrchestrator: cancel_event = cancel_event, stats_holder = stats_holder, read_timeout = _DISPATCH_READ_TIMEOUT, + mark_started = False, ) finally: + # Normally already retired by the dispatcher at gen_done; this covers streams that + # end without one (cancel, disconnect, a dead subprocess). + self._release_worker(cancel_event) with self._mailbox_lock: self._mailboxes.pop(request_id, None) + self._request_cancel_events.pop(request_id, None) def _drain_mailbox( self, @@ -1578,6 +1715,11 @@ class InferenceOrchestrator: # Won the lock handoff during a switch; don't start on the outgoing model. yield GenStreamError("Error: model is being unloaded", public = True) return + if cancel_event is not None and cancel_event.is_set(): + # Stopped while queued on the lock. Sending anyway occupied the worker with a + # run the user ended: the cancel is only seen on a token, so a long prefill + # (or a generation that reaches gen_done without one) held up its siblings. + return request_id = str(uuid.uuid4()) image_b64 = self._pil_to_base64(image) if image is not None else None cmd = self._build_generate_cmd( @@ -1599,22 +1741,95 @@ class InferenceOrchestrator: preserve_thinking = preserve_thinking, ) + # Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the + # lock above, having generated nothing -- cannot reset the generation this is starting. + # Claiming after the send left the command running unclaimed. Released in the finally. + # Own mailbox: a compare request can start the dispatcher while this is streaming, + # and it would otherwise consume our responses and drop them. + read_one, drain, release_mailbox = self._direct_reader(request_id) try: - self._send_cmd(cmd) - except RuntimeError as exc: - yield GenStreamError(f"Error: {exc}") - return + try: + with self._send_order_lock: + self._claim_worker(cancel_event) + self._send_cmd(cmd) + except RuntimeError as exc: + yield GenStreamError(f"Error: {exc}") + return - yield from self._consume_token_stream( - self._read_resp, - lambda: self._drain_until_gen_done(timeout = 5.0), - crash_context = "generation", - cancel_event = cancel_event, - stats_holder = stats_holder, - ) + yield from self._consume_token_stream( + read_one, + lambda: drain(timeout = 5.0), + crash_context = "generation", + cancel_event = cancel_event, + stats_holder = stats_holder, + ) + finally: + self._release_worker(cancel_event) + release_mailbox() - def reset_generation_state(self): - """Cancel any ongoing generation and reset state.""" + def _claim_worker(self, cancel_event) -> None: + """Record this request as one the worker will run. + + Admission only. The subprocess executes generations one at a time, so a + dispatched request sitting behind another in the command queue is claimed + but not executing, and must not be able to signal the shared cancel event + (that would end whichever request IS executing). _mark_worker_started + promotes it once the worker answers it. + """ + with self._active_cancel_lock: + self._active_cancel_events.append(cancel_event) + + def _mark_worker_started(self, cancel_event) -> None: + """Promote a claimed request to executing, on its first worker response. + + Sole executor: the subprocess runs one generation at a time, so answering + this one means it has left the previous one behind. + """ + if cancel_event is None: + return + with self._active_cancel_lock: + if self._executing_cancel_events[:1] != [cancel_event]: + self._executing_cancel_events[:] = [cancel_event] + + def _release_worker(self, cancel_event) -> None: + with self._active_cancel_lock: + for bucket in (self._active_cancel_events, self._executing_cancel_events): + try: + bucket.remove(cancel_event) + except ValueError: + pass + + def _owns_worker(self, cancel_event) -> bool: + """Whether a reset from this request may signal the shared cancel event. + + True when it is one of the EXECUTING generations, and when nothing is in + flight at all: an error path that resets before anything started has no + one else to interrupt, so it must not become a silent no-op. Claimed but + queued does not count, or a Stop on a queued request would end the + running one, including during the prefill before any response arrives. + """ + with self._active_cancel_lock: + if not self._active_cancel_events: + # Nothing in flight at all, so there is no one to protect. + return True + if self._executing_cancel_events: + return any(ev is cancel_event for ev in self._executing_cancel_events) + # Claimed but nothing has answered yet (A is in prefill). The worker takes commands + # in order, so the oldest claim is the executor; anyone else here is queued behind it. + return self._active_cancel_events[0] is cancel_event + + def reset_generation_state(self, caller_cancel_event = None): + """Cancel any ongoing generation and reset state. + + ``caller_cancel_event`` scopes the reset to one request. The worker has a + single cancel event and generation is serialized on _gen_lock, so a chat + that is still queued has no generation of its own to reset: calling this + from its Stop handler would kill whichever chat currently holds the lock. + Pass the request's own event and the reset is dropped unless that request + is the one running. Omit it for genuinely global resets (unload, switch). + """ + if caller_cancel_event is not None and not self._owns_worker(caller_cancel_event): + return self._cancel_generation() if not self._ensure_subprocess_alive(): return @@ -1673,35 +1888,40 @@ class InferenceOrchestrator: if use_adapter is not None: cmd["use_adapter"] = use_adapter - self._send_cmd(cmd) + # Same shared-queue hazard as _generate_inner: see _direct_reader. + read_one, _drain, release_mailbox = self._direct_reader(request_id) + try: + self._send_cmd(cmd) - deadline = time.monotonic() + 120.0 - while time.monotonic() < deadline: - remaining = max(0.1, deadline - time.monotonic()) - resp = self._read_resp(timeout = min(remaining, 1.0)) + deadline = time.monotonic() + 120.0 + while time.monotonic() < deadline: + remaining = max(0.1, deadline - time.monotonic()) + resp = read_one(timeout = min(remaining, 1.0)) - if resp is None: - if not self._ensure_subprocess_alive(): - raise RuntimeError(self._subprocess_crash_message("audio generation")) - continue + if resp is None: + if not self._ensure_subprocess_alive(): + raise RuntimeError(self._subprocess_crash_message("audio generation")) + continue - rtype = resp.get("type", "") + rtype = resp.get("type", "") - if rtype == "audio_done": - wav_bytes = base64.b64decode(resp["wav_base64"]) - sample_rate = resp["sample_rate"] - return wav_bytes, sample_rate + if rtype == "audio_done": + wav_bytes = base64.b64decode(resp["wav_base64"]) + sample_rate = resp["sample_rate"] + return wav_bytes, sample_rate - if rtype == "audio_error": - raise RuntimeError(resp.get("error", "Audio generation failed")) + if rtype == "audio_error": + raise RuntimeError(resp.get("error", "Audio generation failed")) - if rtype == "error": - raise RuntimeError(resp.get("error", "Unknown error")) + if rtype == "error": + raise RuntimeError(resp.get("error", "Unknown error")) - if rtype == "status": - continue + if rtype == "status": + continue - raise RuntimeError("Timeout waiting for audio generation (120s)") + raise RuntimeError("Timeout waiting for audio generation (120s)") + finally: + release_mailbox() def generate_whisper_response( self, @@ -1775,6 +1995,9 @@ class InferenceOrchestrator: # Won the lock handoff during a switch; don't start on the outgoing model. yield GenStreamError("Error: model is being unloaded", public = True) return + if cancel_event is not None and cancel_event.is_set(): + # Stopped while queued on the lock, same as _generate_inner. + return request_id = str(uuid.uuid4()) # numpy array -> list for mp.Queue serialization @@ -1797,18 +2020,28 @@ class InferenceOrchestrator: "repetition_penalty": repetition_penalty, } + # Same shared-queue hazard as _generate_inner: see _direct_reader. + read_one, drain, release_mailbox = self._direct_reader(request_id) try: - self._send_cmd(cmd) - except RuntimeError as exc: - yield GenStreamError(f"Error: {exc}") - return + try: + # Claim under the send lock, like _generate_inner: unclaimed, a compare request queued + # behind this looked like the oldest owner, so stopping it killed this one. + with self._send_order_lock: + self._claim_worker(cancel_event) + self._send_cmd(cmd) + except RuntimeError as exc: + yield GenStreamError(f"Error: {exc}") + return - yield from self._consume_token_stream( - self._read_resp, - lambda: self._drain_until_gen_done(timeout = 5.0), - crash_context = "audio input generation", - cancel_event = cancel_event, - ) + yield from self._consume_token_stream( + read_one, + lambda: drain(timeout = 5.0), + crash_context = "audio input generation", + cancel_event = cancel_event, + ) + finally: + self._release_worker(cancel_event) + release_mailbox() # ------------------------------------------------------------------ # Local helpers (no subprocess needed) diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py index 9345ce3f87..b593bc119b 100644 --- a/studio/backend/core/inference/safetensors_agentic.py +++ b/studio/backend/core/inference/safetensors_agentic.py @@ -59,6 +59,7 @@ from core.tool_healing import ( from core.inference.tool_loop_controller import ( ToolLoopController, append_deferred_nudges, + awaiting_approval_status, coerce_tool_arguments, status_for_tool, tool_event_provenance, @@ -1209,18 +1210,30 @@ def run_safetensors_tool_loop( start_event["awaiting_confirmation"] = needs_confirm try: - yield {"type": "status", "text": decision.status_text} + # A gated call has not started: say waiting, not "Running" (GGUF parity). + yield { + "type": "status", + "text": ( + awaiting_approval_status(decision.tool_name) + if needs_confirm + else decision.status_text + ), + } yield start_event - if ( - decision_slot is not None - and wait_tool_decision( + _decision = ( + wait_tool_decision( decision_slot, approval_id, cancel_event = cancel_event, ) - == "deny" - ): + if decision_slot is not None + else None + ) + if _decision is not None and _decision != "deny": + # Approved: now it really is running. + yield {"type": "status", "text": decision.status_text} + if _decision == "deny": decision_slot = None if provisional_match: provisional_resolved = True diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py index 361f4b20e3..feedae5874 100644 --- a/studio/backend/core/inference/tool_loop_controller.py +++ b/studio/backend/core/inference/tool_loop_controller.py @@ -238,6 +238,19 @@ def status_for_tool(tool_name: str, arguments: Mapping[str, Any]) -> str: return f"Calling: {tool_name}" +def awaiting_approval_status(tool_name: str) -> str: + """Status text for a call parked on the approval prompt. + + It has not started, so reporting "Running ..." with a climbing timer reads + as a hang. + """ + if tool_name == "python": + return "Waiting for approval: Python" + if tool_name == "terminal": + return "Waiting for approval: command" + return f"Waiting for approval: {tool_name}" + + def is_tool_error(result: str) -> bool: return isinstance(result, str) and result.lstrip().startswith(TOOL_ERROR_PREFIXES) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index bd5322819e..0c6e2292bc 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -3105,6 +3105,22 @@ def is_always_safe_tool(name: str) -> bool: return name in _ALWAYS_SAFE_TOOLS +# Tools whose provisional card is only a text preview of the arguments, so it can stream +# while awaiting approval. +_TEXT_PREVIEW_TOOLS = frozenset({"python", "terminal"}) + + +def has_text_only_provisional_card(name: str) -> bool: + """True when streaming this tool's arguments before approval shows only text. + + A large code payload takes a minute or more to write, and suppressing the + card until the call completes leaves the chat blank the whole time. Nothing + runs before the decision either way, and you have to read the code to make + it. + """ + return name in _TEXT_PREVIEW_TOOLS + + def is_potentially_unsafe_tool_call(name: str, arguments: dict) -> bool: """Whether a tool call must still pause for approval in auto mode. diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index fe59bc3e78..e66adb789e 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -191,12 +191,26 @@ class LoadRequest(BaseModel): "auth, UI/server mode) are rejected. Ignored for non-GGUF models." ), ) + force_cancel_active: bool = Field( + False, + description = ( + "Stop chats still generating instead of refusing with 409. A load " + "replaces the llama-server every open conversation decodes on." + ), + ) class UnloadRequest(BaseModel): """Request to unload a model""" model_path: str = Field(..., description = "Model identifier to unload") + force_cancel_active: bool = Field( + False, + description = ( + "Stop chats still generating instead of refusing with 409. An " + "unload takes away the llama-server they are decoding on." + ), + ) class TranscribeRequest(BaseModel): @@ -350,6 +364,14 @@ class InstallLatestTransformersRequest(BaseModel): description = "Exact transformers version to install; must match the current " "latest PyPI release reported by /validate.", ) + force_cancel_active: bool = Field( + False, + description = ( + "Stop chats still generating instead of refusing with 409. The install " + "is a step of the model swap that raised the same prompt, so a client " + "that already got consent for that swap can carry it through here." + ), + ) class InstallLatestTransformersResponse(BaseModel): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9843dc6378..97149f7a17 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1796,6 +1796,7 @@ from core.inference.anthropic_compat import ( AnthropicPassthroughEmitter, ) from auth.authentication import get_current_subject +from state import active_generations from state.tool_approvals import resolve_tool_decision from core.inference.key_exchange import decrypt_api_key @@ -2246,11 +2247,38 @@ def _prune_pending(now: float) -> None: class _TrackedCancel: - """Register cancel_event in _CANCEL_REGISTRY for the block's duration.""" + """Register cancel_event in _CANCEL_REGISTRY for the block's duration. - def __init__(self, event: threading.Event, *keys): + Also records the run in state.active_generations so /load and /unload can + see which chats a reload would interrupt. Both registries share this event, + so either one cancels down the same per-request path. + """ + + def __init__( + self, + event: threading.Event, + *keys, + thread_id = None, + model = None, + kind = "chat", + ): self.event = event self.keys = tuple(k for k in keys if k) + # kind reaches the swap prompt: embeddings and raw completions have no conversation, so + # naming them chats would offer to stop something the user never started from a thread. + self._active = active_generations.ActiveGeneration( + event, thread_id = thread_id, model = model, kind = kind + ) + + @classmethod + def for_payload(cls, event: threading.Event, payload, *keys): + """Track the run against the conversation its request names.""" + return cls( + event, + *keys, + thread_id = getattr(payload, "thread_id", None), + model = getattr(payload, "model", None), + ) def __enter__(self): # Register + consume-pending in one critical section to close the @@ -2264,6 +2292,7 @@ class _TrackedCancel: for k in self.keys: if k and _PENDING_CANCELS.pop(k, None) is not None: should_cancel = True + self._active.__enter__() if should_cancel: self.event.set() return self.event @@ -2277,6 +2306,7 @@ class _TrackedCancel: bucket.discard(self.event) if not bucket: _CANCEL_REGISTRY.pop(k, None) + self._active.__exit__(*exc) return False @@ -3502,15 +3532,38 @@ def _switch_waiter_count() -> int: return sum(max(0, count) for count in _auto_switch_waiters.values()) -async def _wait_for_model_switch_idle(*, current_request_counted: bool) -> None: +async def _wait_for_model_switch_idle( + *, + current_request_counted: bool, + cancel_pending: bool = False, + timeout_s: Optional[float] = None, +) -> None: """Wait until a model replacement cannot interrupt active inference. The caller holds ``inference_lifecycle_gate``, which prevents new inference from starting while existing requests drain. Auto-switch requests that have resolved their targets are scheduler waiters, not active generations, so exclude them to avoid a queue deadlock. + + ``cancel_pending`` is set by a forced swap that has NOT cancelled yet: the + registered generations are the ones it is about to stop, so waiting on them + would wait out exactly what the force exists to end. Excluding them lets the + drain finish ahead of the cancel, which keeps every check that can still + reject the swap in front of the destructive step. Recomputed each poll (not + snapshotted) so a generation that ends on its own stops being discounted and + the remaining, non-cancellable requests are still waited out. + + ``timeout_s`` bounds the wait and returns rather than raising. Only the + post-cancel drains pass it: what they wait on may never observe its cancel + (TTS on the subprocess backend has no observer), and they hold the lifecycle + gate, so an unbounded wait pins every load and unload behind one + uninterruptible generation. Expiring there just proceeds, which is what they + do anyway once drained. Pre-cancel drains stay unbounded -- the swap can + still be refused, so they must not shorten the protection they provide. """ from core.inference.llama_keepwarm import other_inference_request_count + + deadline = None if timeout_s is None else time.monotonic() + timeout_s while True: queued_switches = _switch_waiter_count() if current_request_counted and queued_switches > 0: @@ -3519,8 +3572,19 @@ async def _wait_for_model_switch_idle(*, current_request_counted: bool) -> None: current_request_counted = current_request_counted, include_pending = False, ) + if cancel_pending: + active_others -= min(active_others, active_generations.count()) if active_others <= queued_switches: return + if deadline is not None and time.monotonic() >= deadline: + logger.warning( + "model_switch_drain_timed_out", + extra = { + "event": "inference.switch_drain_timeout", + "remaining": active_others - queued_switches, + }, + ) + return await asyncio.sleep(0.02) @@ -4796,6 +4860,214 @@ def _raise_if_sidecar_swap_in_progress() -> None: ) +def _raise_or_cancel_active_generations( + *, + force: bool, + action: str, + cancel: bool = True, +) -> int: + """Gate a model swap on the chats currently generating. + + Every open conversation decodes on the single llama-server this route is + about to replace, so refuse with 409 and name them. force_cancel_active + instead stops them through the same events an explicit Stop uses. Returns + how many were cancelled. The frontend guard is bypassable from a second tab + or curl; this one is not. + + ``cancel = False`` runs the refusal half only. /load calls it that way once + up front, so a non-forced swap still fails fast, and again with cancel just + before teardown: cancelling is destructive and unrecoverable, so it must not + run ahead of preflight checks that can still reject the load (see + _load_model_impl). + """ + if not active_generations.count(): + return 0 + if not force: + thread_ids = active_generations.active_thread_ids() + running = active_generations.count() + raise HTTPException( + status_code = 409, + detail = { + "error": "active_generations", + "message": ( + f"{action} would stop {running} chat" + f"{'s' if running != 1 else ''} that " + f"{'are' if running != 1 else 'is'} still generating. " + "Stop them first, or retry with force_cancel_active." + ), + "running": running, + "thread_ids": thread_ids, + }, + ) + if not cancel: + # Refusal-only pass: the caller cancels later, once nothing can still reject the load. + return 0 + cancelled = active_generations.cancel_all() + if cancelled: + logger.info( + "model_swap_cancelled_active_generations", + extra = {"event": "inference.reload_cancelled_generations", "count": cancelled}, + ) + return cancelled + + +_POST_CANCEL_DRAIN_TIMEOUT_S = 5.0 + + +async def _cancel_and_drain_for_sidecar_swap(timeout_s: Optional[float] = None) -> None: + """Clear the way for a confirmed sidecar swap, then stop the chats it interrupts. + + The installer gates on the middleware's in-flight count, not on + active_generations, so it also sees requests the cancel cannot stop. Drain + those FIRST, discounting the registered chats (they are what the cancel is + for, so waiting on them would wait out the point of the force). Only then + cancel, and let the survivors unwind. Cancelling first meant an unrelated + counted request -- a /v1/messages/count_tokens, say -- was still there for + the caller's recheck, which then refused an install that had already stopped + every chat for nothing. + + Bounded on both halves: the requests being waited on may never observe a + cancel, and this holds the lifecycle gate and the sidecar reservation inside + ``asyncio.shield``, so an unbounded wait would wedge the process. Expiring in + the first half returns without cancelling, so the caller's recheck refuses + with the chats untouched. + """ + from core.inference.llama_keepwarm import other_inference_request_count + + budget = _POST_CANCEL_DRAIN_TIMEOUT_S if timeout_s is None else timeout_s + + async def _drain(deadline: float, *, discount_registered: bool) -> bool: + while True: + counted = other_inference_request_count( + current_request_counted = False, include_pending = False + ) + if discount_registered: + counted -= min(counted, active_generations.count()) + if counted <= 0: + return True + if time.monotonic() >= deadline: + return False + await asyncio.sleep(0.02) + + # Weighted, not halved, so the total wait under the gate is unchanged. The first drain only + # asks whether unrelated inference is in flight; cutting the second short refused installs + # whose chats had already been stopped for nothing. + if not await _drain(time.monotonic() + budget / 5, discount_registered = True): + return + _raise_or_cancel_active_generations(force = True, action = "Installing a new transformers version") + await _drain(time.monotonic() + budget * 4 / 5, discount_registered = False) + + +async def _drain_and_recancel_before_teardown(*, force: bool, action: str) -> None: + """Wait out inference the registry cannot see, then stop anything new. + + A request that passed the keep-warm middleware but has not reached its + ``_TrackedCancel`` yet is counted in-flight and absent from the registry, so + cancelling on the registry alone lets a teardown land on an already-admitted + request. Drain on the middleware count instead, which covers both the runs + just cancelled and the ones still in that window, then cancel again for + anything that registered while waiting. + + Bounded and non-raising: an unload is a deliberate user action, so the worst + case stays what it is today rather than becoming a refusal. + """ + await _wait_for_model_switch_idle( + current_request_counted = False, + timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S, + ) + if force: + _raise_or_cancel_active_generations(force = True, action = action) + + +_UNRESOLVED_BACKEND_STATE = object() + + +def _unload_evicts_standard_backend(backend, model_path: str) -> bool: + """Whether ``backend.unload_model(model_path)`` will really evict something. + + The standard backend refuses to unload a name it never loaded ("don't unload + a stale model") and returns success, so /unload for a model another tab has + already replaced is a no-op. That must not count as a teardown: cancelling + the running chats for it would end them and leave the resident model up. + + Mirrors the backend's own guard (case-insensitive on the active name, since + the load path canonicalizes casing). A backend that exposes neither field is + reported as a real unload, which keeps the previous behaviour. + """ + active = getattr(backend, "active_model_name", _UNRESOLVED_BACKEND_STATE) + loaded = getattr(backend, "models", _UNRESOLVED_BACKEND_STATE) + if active is _UNRESOLVED_BACKEND_STATE and loaded is _UNRESOLVED_BACKEND_STATE: + return True + if isinstance(active, str) and active and active.lower() == (model_path or "").lower(): + return True + return isinstance(loaded, dict) and model_path in loaded + + +def _unload_may_evict(model_path: str) -> bool: + """Whether POST /unload for ``model_path`` can still tear something down. + + The refusal passes gate on this. A request naming a model another tab has + already replaced reaches none of the teardown branches and returns the + documented idempotent no-op (see _unload_evicts_standard_backend), so + refusing it counts a teardown that cannot happen and leaves a stale tab + unable to clear its selection. Each disjunct mirrors one teardown branch, so + True means "some branch may fire", never "this unload succeeds". + + Attribute reads only, no lifecycle gate, so the pre-gate pass still fails + fast on a swap that would really stop chats. A stale answer is safe in both + directions: the gated pass re-runs this under the gate, and every branch + re-runs the refusal at its own point of no return, so a False here can never + let a teardown through unrefused. + """ + backend = get_inference_backend() + loading = getattr(backend, "get_loading_model", lambda: None)() + if ( + loading is not None + and hasattr(backend, "cancel_load") + and (model_path == loading or model_path.lower() == loading.lower()) + ): + return True + llama_backend = get_llama_cpp_backend() + if llama_backend.is_active and ( + llama_backend.model_identifier == model_path + or is_registered_native_path_label(llama_backend.model_identifier, model_path) + # Up but not serving is mid-load, evicted whatever model was named. + or not llama_backend.is_loaded + ): + return True + return _unload_evicts_standard_backend(backend, model_path) + + +@studio_router.get("/active-generations") +async def get_active_generations( + fastapi_request: Request, current_subject: str = Depends(get_current_subject) +): + """Conversations currently generating, plus how many can decode at once. + + Lets a model swap name the chats it would interrupt, including runs this tab + cannot see (another tab, or a reload behind a proxy). parallel_slots is the + slot count actually in use, which the VRAM fit may have cut below the + requested --parallel; chats beyond it queue rather than fail. + """ + entries = active_generations.snapshot() + # A tracker's model can be a native local path (the legacy stream records active_model_name + # verbatim); redact here, the one place that serialises it. + for _entry in entries: + if isinstance(_entry.get("model"), str): + _entry["model"] = redact_native_paths(_entry["model"]) + slots = 1 + try: + slots = _openai_llama_admission_capacity(fastapi_request, get_llama_cpp_backend()) + except Exception: + slots = int(getattr(fastapi_request.app.state, "llama_parallel_slots", 1) or 1) + return { + "active": entries, + "count": len(entries), + "thread_ids": active_generations.active_thread_ids(), + "parallel_slots": max(1, int(slots)), + } + + @router.post("/load", response_model = LoadResponse) async def load_model( request: LoadRequest, @@ -4823,7 +5095,18 @@ async def load_model( # holds this gate. async with inference_lifecycle_gate(): _raise_if_sidecar_swap_in_progress() - return await _load_model_impl(request, fastapi_request, current_subject) + # The active-generation gate runs inside _load_model_impl, once it knows this is a real + # reload, and still under the lifecycle gate so the check stays atomic with the teardown. + return await _load_model_impl( + request, + fastapi_request, + current_subject, + on_reload_confirmed = lambda *, cancel: _raise_or_cancel_active_generations( + force = request.force_cancel_active, + action = "Loading a model", + cancel = cancel, + ), + ) async def _load_model_impl( @@ -4832,6 +5115,7 @@ async def _load_model_impl( current_subject: str, *, current_request_counted: bool = False, + on_reload_confirmed = None, ): from core.inference.llama_cpp import LlamaServerNotFoundError @@ -5041,6 +5325,19 @@ async def _load_model_impl( chat_template = _chat_template, ) + # Past every already_loaded fast return, so this really will replace the running model: gate + # it on the chats that would stop. Refusal only, so a non-forced swap fails fast; the checks + # between here and the teardown (identifier, GPU, training guard, downloads) can still + # reject the load, and cancelling now would stop every chat for a model that never loads. + # Auto-switch passes no hook and keeps its current behaviour. + if on_reload_confirmed is not None: + on_reload_confirmed(cancel = False) + + # Destructive cancel still owed at the teardown below, so it can be deferred past every + # remaining check; the drains key off this. Only a forced swap cancels: unforced already + # 409'd above, auto-switch has no hook. + cancel_pending = on_reload_confirmed is not None and bool(request.force_cancel_active) + # is_lora auto-detected from adapter_config.json on disk/HF. # DNS-probe wrap so offline loads skip 30-60s of soft-failed network # checks before the worker starts. @@ -5154,13 +5451,33 @@ async def _load_model_impl( ), ) - # Keep the resident model alive until every active generation finishes; - # the caller's lifecycle gate blocks new starts. - await _wait_for_model_switch_idle(current_request_counted = current_request_counted) - # A sidecar install can reserve the gate while inference drains, after the - # route-level checks above, so recheck before replacing either backend. + # Fast path only: a swap can still be reserved during the drain. _raise_if_sidecar_swap_in_progress() + # Drain active generations first (the lifecycle gate blocks new starts); a forced swap + # excludes the ones it is about to cancel rather than waiting them out. + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + cancel_pending = cancel_pending, + ) + # Decisive recheck, and the last thing that can reject this load, so it runs BEFORE the + # cancel: rejecting after would stop every chat for nothing. + _raise_if_sidecar_swap_in_progress() + + # Point of no return for the GGUF path: nothing left can reject this load, so stop the + # chats the swap interrupts (or refuse, if the caller never opted in). + if on_reload_confirmed is not None: + on_reload_confirmed(cancel = True) + + # Let the cancelled generations unwind before the teardown; no check follows, so this cannot + # strand a cancelled chat behind a 409. Bounded: TTS observes no cancel event, so an + # unbounded wait would hold the gate for a whole audio run. + if cancel_pending: + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S, + ) + # Unload any active Unsloth model only after every hub conflict check. if unsloth_backend.active_model_name: logger.info( @@ -5376,10 +5693,27 @@ async def _load_model_impl( # ── Standard path: load via Unsloth/transformers ────────── backend = get_inference_backend() - # Unload any active GGUF model first - llama_backend = get_llama_cpp_backend() - await _wait_for_model_switch_idle(current_request_counted = current_request_counted) + # Same sidecar rejection as GGUF: fast path ahead of the drain, rechecked after. _raise_if_sidecar_swap_in_progress() + + llama_backend = get_llama_cpp_backend() + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + cancel_pending = cancel_pending, + ) + _raise_if_sidecar_swap_in_progress() + + # Point of no return for the Unsloth path: cancel only once nothing can still reject the load. + if on_reload_confirmed is not None: + on_reload_confirmed(cancel = True) + + # Let the cancelled generations unwind before the teardown; no check follows. Bounded like GGUF. + if cancel_pending: + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S, + ) + # Unload any active GGUF model first if llama_backend.is_loaded: logger.info("Unloading GGUF model before loading Unsloth model") llama_backend.unload_model() @@ -5978,7 +6312,13 @@ async def install_latest_transformers_route( other_inference_request_count, ) - if other_inference_request_count(current_request_counted = False, include_pending = False) > 0: + # A confirmed swap skips only this fast path; the recheck under the gate still has to pass, + # so the guard is unchanged for anyone who did not confirm. + if ( + not request.force_cancel_active + and other_inference_request_count(current_request_counted = False, include_pending = False) + > 0 + ): raise HTTPException( status_code = 409, detail = ( @@ -6072,9 +6412,16 @@ async def install_latest_transformers_route( "Retry the install." ), ) + # Carry a confirmed swap's decision through: the user already accepted the "stop N + # chats" prompt, and refusing here would make that answer unactionable (Retry + # cannot succeed while the same chats run). Deliberately LAST, after every check + # that can still reject the install, so the cancel is spent only once nothing can + # turn this request away -- /load's rule. + if request.force_cancel_active: + await _cancel_and_drain_for_sidecar_swap() # Recheck under the gate: new streams bump their in-flight count while - # holding it, so once held nothing slips past (the pre-gate check is only - # a fast path and can be outlasted by a wait on a long /load). + # holding it, so once held nothing slips past. A forced install that could + # not drain in time lands here too, for the same 409 as without the flag. if ( other_inference_request_count( current_request_counted = False, include_pending = False @@ -6126,9 +6473,9 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge from core.inference.llama_keepwarm import inference_lifecycle_gate, note_model_unloaded try: # "Stop loading" (frontend cancelLoading -> /unload) must abort a still-loading - # model promptly. /load holds the lifecycle gate for the whole (multi-minute) load, - # so gating first would make the cancel wait it out. cancel_load only tears the - # loading subprocess down (no unload command), so it is safe off-gate. + # model promptly, and /load holds the lifecycle gate for the whole load. cancel_load only + # tears the loading subprocess down, so it is safe off-gate -- and ahead of the + # active-generation refusal below, which it can never need (see there). backend = get_inference_backend() loading = getattr(backend, "get_loading_model", lambda: None)() if ( @@ -6141,13 +6488,11 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge logger.info(f"Cancelled in-flight load: {request.model_path}") return UnloadResponse(status = "unloaded", model = request.model_path) - # Same "stop loading" fast path for a still-loading GGUF (llama-server spawned, - # health check not yet passed). A gated unload would wait out the multi-minute - # load; unload_model() sets the cancel_event load_model polls off its own lock and - # kills the child, sending no worker command, so it is safe off-gate like - # cancel_load. The gated GGUF branch below handles the already-loaded case. Gate on - # the loading model (identifier or native label): the single llama-server loads one - # GGUF at a time, so an unload for a different model must not cancel this load. + # Same "stop loading" fast path for a still-loading GGUF (spawned, health check not passed). + # unload_model() sets the cancel_event load_model polls and kills the child without a + # worker command, so it is safe off-gate like cancel_load; the gated branch below handles + # the already-loaded case. Gated on the loading model so an unload for a different model + # cannot cancel this load. llama_backend = get_llama_cpp_backend() if ( llama_backend.is_active @@ -6164,11 +6509,35 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge logger.info(f"Cancelled in-flight GGUF load: {request.model_path}") return UnloadResponse(status = "unloaded", model = request.model_path) + # Same gate as /load: refusal only, so a non-forced unload fails fast before queueing on the + # lifecycle gate. Skipped when no teardown branch can fire, or a request naming a model + # another tab already replaced would 409 on chats it cannot interrupt. + # + # BEHIND the two "stop loading" fast paths above: both cancel a load that has not replaced + # anything yet, so neither can interrupt a chat, and refusing them counted a teardown that + # cannot happen (unretryably -- the frontend's Cancel sends this unload unforced and drops + # the error). Any other name still falls through here. + if _unload_may_evict(request.model_path): + _raise_or_cancel_active_generations( + force = request.force_cancel_active, + action = "Unloading the model", + cancel = False, + ) + # Serialize with /load under the same lifecycle gate: the Unsloth unload now runs # off the event loop (asyncio.to_thread), so without this a concurrent /load could # swap in a fresh subprocess mid-unload and the unload command would land on the # new worker. The gate makes load and unload exclusive. async with inference_lifecycle_gate(): + # Rechecked under the gate, like /load: a chat can register while this one queues here (the + # middleware takes and releases the same gate). Still refusal only, and re-read rather + # than carried down, since a load may have finished meanwhile. + if _unload_may_evict(request.model_path): + _raise_or_cancel_active_generations( + force = request.force_cancel_active, + action = "Unloading the model", + cancel = False, + ) # Check if the GGUF backend has this model loaded or is loading it. llama_backend = get_llama_cpp_backend() if llama_backend.is_active and ( @@ -6181,8 +6550,18 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge # Read the identity before teardown clears it, so the row reads repo:QUANT. _unloaded = _llama_public_model_id(llama_backend, request.model_path) _unloaded_variant = getattr(llama_backend, "hf_variant", None) - # A manual unload is a deliberate user action: tear down now even if a - # request is mid-stream (only the automatic idle loop defers to it). + # Point of no return: this really does replace the running server, so stop the + # chats. A manual unload is a deliberate user action, so it cancels mid-stream + # requests rather than deferring to them the way the automatic idle loop does. + _raise_or_cancel_active_generations( + force = request.force_cancel_active, action = "Unloading the model" + ) + # Let what we just cancelled unwind first, like /load: tearing the server down under + # streams told to stop but not yet finished turned a clean end into a dropped + # connection. Bounded, since a manual unload is deliberate. + await _drain_and_recancel_before_teardown( + force = request.force_cancel_active, action = "Unloading the model" + ) llama_backend.unload_model() note_model_unloaded() api_monitor.record_lifecycle( @@ -6197,6 +6576,14 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge # a slow SSE stream paused between tokens still holds, so a sync call would block # the loop that drives the stream's next token and the lock release. backend = get_inference_backend() + if _unload_evicts_standard_backend(backend, request.model_path): + # Point of no return for the standard path, same rule as above. + _raise_or_cancel_active_generations( + force = request.force_cancel_active, action = "Unloading the model" + ) + await _drain_and_recancel_before_teardown( + force = request.force_cancel_active, action = "Unloading the model" + ) await asyncio.to_thread(backend.unload_model, request.model_path) note_model_unloaded() api_monitor.record_lifecycle( @@ -6207,6 +6594,9 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge logger.info(f"Unloaded model: {request.model_path}") return UnloadResponse(status = "unloaded", model = request.model_path) + except HTTPException: + # Typed refusals (the gate's 409) must not be rewritten as a 500 below. + raise except Exception as e: logger.error(f"Error unloading model: {e}", exc_info = True) raise HTTPException(status_code = 500, detail = "Failed to unload model") @@ -6355,6 +6745,12 @@ async def generate_stream( disconnect_watcher = asyncio.create_task( _await_disconnect_then_cancel(fastapi_request, cancel_event) ) + # Registered inside the generator, under the finally that unregisters it, so a response whose + # body never starts leaves nothing behind. Unregistered, this run passes /unload's 409 gate + # (which runs no idle drain) and a forced swap has no event to signal. GenerateRequest + # carries no thread_id: counted, not nameable. + _tracker = _TrackedCancel(cancel_event, model = backend.active_model_name) + _tracker.__enter__() try: gen = backend.generate_chat_response( messages = request.messages, @@ -6375,7 +6771,7 @@ async def generate_stream( # Watcher set cancel_event between chunks. Reset here: closing # the generator does not signal a subprocess backend, so it would # keep decoding. The finally's reset is guarded, so no double-run. - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) break chunk = await asyncio.to_thread(next, gen, _DONE) if chunk is _DONE: @@ -6391,24 +6787,28 @@ async def generate_stream( except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) raise except Exception as e: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) logger.error(f"Error during generation: {e}", exc_info = True) yield f"data: {json.dumps({'error': _friendly_error(e)})}\n\n" yield "data: [DONE]\n\n" finally: - await _stop_local_disconnect_cancel_watcher(disconnect_watcher) - if not completed and not cancel_event.is_set(): - cancel_event.set() - backend.reset_generation_state() - if gen is not None: - try: - await asyncio.to_thread(gen.close) - except (RuntimeError, ValueError): - pass + # Nested so a teardown failure still unregisters; a phantom entry 409s swaps. + try: + await _stop_local_disconnect_cancel_watcher(disconnect_watcher) + if not completed and not cancel_event.is_set(): + cancel_event.set() + backend.reset_generation_state(cancel_event) + if gen is not None: + try: + await asyncio.to_thread(gen.close) + except (RuntimeError, ValueError): + pass + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(stream()) @@ -6675,6 +7075,10 @@ async def generate_audio( # the idle-stash restore runs here; switching TTS models is an explicit /load. await _maybe_auto_switch_model(_RELOAD_ONLY_MODEL, request, current_subject) + # Created before the backend pick so the GGUF lambda can close over it; the registration + # that arms it is below, once the model name is known. + _audio_cancel = threading.Event() + # Pick backend — both return (wav_bytes, sample_rate) llama_backend = get_llama_cpp_backend() if llama_backend.is_loaded and getattr(llama_backend, "_is_audio", False): @@ -6691,6 +7095,7 @@ async def generate_audio( min_p = payload.min_p, max_new_tokens = _effective_max_tokens(payload) or 2048, repetition_penalty = payload.repetition_penalty, + cancel_event = _audio_cancel, ) else: backend = get_inference_backend() @@ -6719,11 +7124,30 @@ async def generate_audio( # /audio/generate route and the chat-completions audio branches that delegate here. _fill_recommended_sampling_openai(payload, _audio_model_id) - try: - wav_bytes, sample_rate = await asyncio.to_thread(gen) - except Exception as e: - logger.error(f"Audio generation error: {e}", exc_info = True) - raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + # TTS holds the model for the whole request, so unregistered a non-forced swap counted zero + # generations and tore the model down mid-generation. The GGUF path observes the event; the + # subprocess backend blocks on its response queue with no cancel plumbing, so there it is + # only advisory -- which is why the swap drains are bounded. No cancel keys: /cancel + # addresses streams, and this route has none. + with _TrackedCancel( + _audio_cancel, + thread_id = getattr(payload, "thread_id", None), + model = model_name, + kind = "audio", + ): + # Stop in the UI aborts the fetch and nothing more, and this route has no cancel id to + # address, so without watching the disconnect llama-server kept generating for the rest + # of the request timeout after the chat had already reported it stopped. + _audio_watcher = asyncio.create_task(_await_disconnect_then_cancel(request, _audio_cancel)) + try: + wav_bytes, sample_rate = await asyncio.to_thread(gen) + except Exception as e: + if _audio_cancel.is_set(): + raise HTTPException(status_code = 499, detail = "Audio generation cancelled") + logger.error(f"Audio generation error: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + finally: + await _stop_local_disconnect_cancel_watcher(_audio_watcher) audio_b64 = base64.b64encode(wav_bytes).decode("ascii") return JSONResponse( @@ -8415,7 +8839,7 @@ async def openai_chat_completions( if payload.stream: _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() async def audio_input_stream(): @@ -8481,6 +8905,12 @@ async def openai_chat_completions( }, ) else: + # `stream` defaults to False, so this is the ordinary shape of an audio-input chat and it + # holds the worker for the whole request. Unregistered, a swap counted zero generations + # and cancelled it instead of 409ing (/unload runs no idle drain). + _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) + _tracker.__enter__() try: full_text = "" for chunk_text in audio_input_generate(): @@ -8494,6 +8924,9 @@ async def openai_chat_completions( except Exception as e: api_monitor.fail(monitor_id, _friendly_error(e)) raise + finally: + # Nested under the except arms too: api_monitor.fail() can throw, and a leaked entry 409s swaps. + _tracker.__exit__(None, None, None) api_monitor.set_reply(monitor_id, full_text) api_monitor.finish(monitor_id) response = ChatCompletion( @@ -8652,7 +9085,7 @@ async def openai_chat_completions( monitor_id = monitor_id, ) _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() try: return await _openai_passthrough_non_streaming( @@ -8889,13 +9322,37 @@ async def openai_chat_completions( _tool_sentinel = object() _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() async def gguf_tool_stream(): gen = None next_task = None stream_completed = False + # A call parked on the approval prompt is not decoding, so it gives its slot back; + # otherwise unanswered prompts hold every slot. + _parked = False + + async def _park_admission(on: bool, *, wait: bool = True): + nonlocal _parked + if on == _parked: + return + # This run's own lease, not a fresh lookup: queues are keyed by base_url and a + # reload mints a new port, so re-resolving could release someone else's slot. + lease = reservation.lease_nowait() + if lease is None: + return + if on: + lease.park() + elif wait: + # Resuming: park() may have handed our slot to a waiter, so wait for room instead + # of putting two holders on one slot. + await lease.unpark_async(cancel_event = cancel_event) + else: + # Tearing down; the lease is released separately. + lease.unpark() + _parked = on + disconnect_watcher = asyncio.create_task( _await_disconnect_then_cancel(request, cancel_event) ) @@ -8955,6 +9412,12 @@ async def openai_chat_completions( if event is _tool_sentinel: break + # Anything after the gated tool_start means the user answered. + if not ( + event["type"] == "tool_start" and event.get("awaiting_confirmation") + ): + await _park_admission(False) + if event["type"] == "heartbeat": # Tool-wrapper heartbeat while a server-side tool blocks; keeps SSE alive. yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE @@ -8993,6 +9456,8 @@ async def openai_chat_completions( yield chunk prev_text = "" reasoning_extractor = _new_chat_reasoning_extractor() + # Yielded just before the loop blocks on the user. + await _park_admission(bool(event.get("awaiting_confirmation"))) yield f"data: {json.dumps(event)}\n\n" continue @@ -9076,6 +9541,8 @@ async def openai_chat_completions( error_chunk = _openai_stream_error_chunk(e) yield _openai_stream_error_sse(error_chunk) finally: + # A disconnect mid-approval must not leave a slot parked. + await _park_admission(False, wait = False) try: if not stream_completed: cancel_event.set() @@ -9466,7 +9933,7 @@ async def openai_chat_completions( if _wants_multiple_choices(payload): raise _reject_unsupported_n("streaming GGUF chat completions") _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() try: reservation, admission_config = _openai_llama_admission_reserve( @@ -9785,7 +10252,7 @@ async def openai_chat_completions( raise _openai_admission_http_exception(exc, status_code = 429) _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() admission_lease = None admission_wait_started_at = None @@ -10227,7 +10694,7 @@ async def openai_chat_completions( _sf_tool_sentinel = object() _sf_cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _sf_tracker = _TrackedCancel(cancel_event, *_sf_cancel_keys) + _sf_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_sf_cancel_keys) _sf_tracker.__enter__() async def sf_tool_stream(): @@ -10256,11 +10723,11 @@ async def openai_chat_completions( while True: if cancel_event.is_set(): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) break if await request.is_disconnected(): cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") return @@ -10283,7 +10750,7 @@ async def openai_chat_completions( if event is _sf_tool_sentinel: break if isinstance(event, GenStreamError): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(event) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse( @@ -10378,16 +10845,16 @@ async def openai_chat_completions( except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") raise except GenStreamErrorRaised as exc: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}}) except Exception: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) # Generic wire message; full trace stays in the log (CWE-209: # transformers/torch errors may leak paths). logger.exception("safetensors tool stream error") @@ -10481,20 +10948,20 @@ async def openai_chat_completions( return _model_json_response(response) except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") raise except GenStreamErrorRaised as exc: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) raise HTTPException(status_code = 500, detail = _msg) except HTTPException as exc: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.fail(monitor_id, str(exc.detail)) raise except Exception: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) # CWE-209: generic detail; full trace in log. logger.exception("safetensors tool completion error") api_monitor.fail(monitor_id, "An internal error occurred.") @@ -10619,7 +11086,7 @@ async def openai_chat_completions( # ── Streaming response ──────────────────────────────────────── if payload.stream: _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() async def stream_chunks(): @@ -10646,7 +11113,7 @@ async def openai_chat_completions( gen = generate() while True: if cancel_event.is_set(): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) break # Stall keepalive (see safetensors tool stream) each window while # next(gen) runs in a worker. next(gen, _DONE) returns _DONE rather @@ -10666,7 +11133,7 @@ async def openai_chat_completions( if cumulative is _DONE: break if isinstance(cumulative, GenStreamError): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(cumulative) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse( @@ -10675,7 +11142,7 @@ async def openai_chat_completions( return if await request.is_disconnected(): cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") return new_text = cumulative[len(prev_text) :] @@ -10776,18 +11243,18 @@ async def openai_chat_completions( except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") raise except GenStreamErrorRaised as exc: # Adapter-controlled (compare-mode) backend failure. Honor the # public flag so operational errors surface their real message. - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}}) except Exception as e: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) logger.error(f"Error during OpenAI streaming: {e}", exc_info = True) _msg = _friendly_error(e) api_monitor.fail(monitor_id, _msg) @@ -10826,11 +11293,17 @@ async def openai_chat_completions( # ── Non-streaming response ──────────────────────────────────── else: + # `stream` defaults to False, so this is the default shape of a standard (non-GGUF) chat and + # generate() holds the worker throughout. Unregistered, a swap cancelled this run rather + # than returning 409 (/unload runs no idle drain). + _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) + _tracker.__enter__() try: full_text = "" for token in generate(): if isinstance(token, GenStreamError): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(token) api_monitor.fail(monitor_id, _msg) raise HTTPException(status_code = 500, detail = _msg) @@ -10937,15 +11410,18 @@ async def openai_chat_completions( except GenStreamErrorRaised as exc: # Adapter-controlled (compare-mode) backend failure. Honor the public # flag so operational errors surface their real message. - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) raise HTTPException(status_code = 500, detail = _msg) except Exception as e: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) logger.error(f"Error during OpenAI completion: {e}", exc_info = True) api_monitor.fail(monitor_id, _friendly_error(e)) raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + finally: + # Nested under the except arms too: reset_generation_state() can throw, and a leaked entry 409s swaps. + _tracker.__exit__(None, None, None) # ===================================================================== @@ -11399,10 +11875,11 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge target_url = f"{llama_backend.base_url}/v1/completions" is_stream = body.get("stream", False) prompt_text = _flatten_monitor_prompt(body.get("prompt", "")) + monitor_model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default") monitor_id = api_monitor.start( endpoint = request.url.path, method = request.method, - model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"), + model = monitor_model, prompt = prompt_text, context_length = llama_backend.context_length, subject = current_subject, @@ -11430,12 +11907,23 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge bytes_iter = None disconnect_event = threading.Event() disconnect_watcher = None + # This proxy relays straight from llama-server, so the swap gate has to see it: without an + # entry a non-forced /unload counts zero generations and tears the server down mid-response. + # Sharing disconnect_event lets a forced swap stop the relay through the check it already + # polls. Entered inside the body generator, so a response whose body never starts leaves + # nothing behind (see _responses_stream). No thread_id: public API surface, not a chat. + _tracker = _TrackedCancel(disconnect_event, model = monitor_model, kind = "completions") + _tracker.__enter__() try: req = client.build_request( "POST", target_url, json = body, headers = {"Connection": "close"} ) first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S - resp = await _send_stream_with_preheader_cancel(client, req, request = request) + # Same event the relay loop polls, so a forced swap ends the request during prefill + # instead of only once headers arrive. + resp = await _send_stream_with_preheader_cancel( + client, req, disconnect_event, request = request + ) if resp is None: api_monitor.finish(monitor_id, "cancelled") return @@ -11506,27 +11994,64 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge yield _openai_stream_error_sse_bytes(error_chunk) return finally: - await _aclose_stream_resources( - watchers = (disconnect_watcher,), - iterator = bytes_iter, - resp = resp, - client = client, - ) + # Nested so a close-time failure still unregisters; a phantom entry 409s swaps. + try: + await _aclose_stream_resources( + watchers = (disconnect_watcher,), + iterator = bytes_iter, + resp = resp, + client = client, + ) + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(_stream()) else: - try: - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _llama_non_streaming_generation_timeout(), + # ``stream`` defaults to false, so this common shape registers with the swap gate like the + # streaming branch: unregistered, a non-forced /unload counts zero generations and kills + # llama-server mid-request, and force_cancel_active has no event. Unpooled client so a + # cancel-close hits this call only. + _cancel_event = threading.Event() + _client = _cancelable_nonstreaming_client() + _tracker = _TrackedCancel(_cancel_event, model = monitor_model, kind = "completions") + _tracker.__enter__() + _cancel_watcher = asyncio.create_task( + _await_cancel_or_disconnect_then_close_client( + cancel_event = _cancel_event, + request = request, + client = _client, ) + ) + try: + try: + resp = await _client.post( + target_url, + json = body, + timeout = _llama_non_streaming_generation_timeout(), + ) + except httpx.RequestError: + # The watcher closed the client out from under the request: report the cancel, not a transport failure. + if _cancel_event.is_set(): + raise asyncio.CancelledError() + raise + if _cancel_event.is_set(): + raise asyncio.CancelledError() except asyncio.CancelledError: api_monitor.finish(monitor_id, "cancelled") raise except Exception as e: api_monitor.fail(monitor_id, _friendly_error(e)) raise + finally: + # Nested so a close-time failure still unregisters; a phantom entry 409s swaps. + try: + await _stop_local_disconnect_cancel_watcher(_cancel_watcher) + try: + await _client.aclose() + except Exception: + pass + finally: + _tracker.__exit__(None, None, None) if resp.status_code != 200: api_monitor.fail(monitor_id, resp.text[:500]) @@ -11622,18 +12147,54 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get subject = current_subject, ) - try: - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S, + # Same gate registration as the completions proxy: unregistered, a non-forced /unload counts + # zero generations and kills llama-server mid-embedding. Unpooled client so a cancel-close + # hits this call only. + _cancel_event = threading.Event() + _client = _cancelable_nonstreaming_client() + _tracker = _TrackedCancel( + _cancel_event, + model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"), + kind = "embeddings", + ) + _tracker.__enter__() + _cancel_watcher = asyncio.create_task( + _await_cancel_or_disconnect_then_close_client( + cancel_event = _cancel_event, + request = request, + client = _client, ) + ) + try: + try: + resp = await _client.post( + target_url, + json = body, + timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S, + ) + except httpx.RequestError: + # The watcher closed the client out from under the request: report the cancel, not a transport failure. + if _cancel_event.is_set(): + raise asyncio.CancelledError() + raise + if _cancel_event.is_set(): + raise asyncio.CancelledError() except asyncio.CancelledError: api_monitor.finish(monitor_id, "cancelled") raise except Exception as exc: api_monitor.fail(monitor_id, _friendly_error(exc)) raise + finally: + # Nested so a close-time failure still unregisters; a phantom entry 409s swaps. + try: + await _stop_local_disconnect_cancel_watcher(_cancel_watcher) + try: + await _client.aclose() + except Exception: + pass + finally: + _tracker.__exit__(None, None, None) if resp.status_code != 200: api_monitor.fail(monitor_id, resp.text[:500]) else: @@ -12366,6 +12927,12 @@ async def _responses_stream( ) body["stream_options"] = {"include_usage": True} target_url = f"{llama_backend.base_url}/v1/chat/completions" + # The stream's own disconnect event, shared with the cancel/active-generation registries: + # this path decodes on llama-server, so a non-forced /unload must see it and refuse instead + # of tearing the server down mid-response. Entered inside the body generator below, so a + # response whose body never starts leaves nothing behind. + cancel_event = threading.Event() + _tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id) try: reservation, admission_config = _openai_llama_admission_reserve( request = request, @@ -12819,14 +13386,19 @@ async def _responses_stream( resp = None lines_iter = None disconnect_watcher = None - disconnect_event = threading.Event() + # Tracked per-run event: a client disconnect and a forced reload both land here. + disconnect_event = cancel_event try: req = client.build_request( "POST", target_url, json = body, headers = {"Connection": "close"} ) first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S try: - resp = await _send_stream_with_preheader_cancel(client, req, request = request) + # Same event the loop below polls: prefill can run for the whole first-token window, + # and only the send watcher can end it early. + resp = await _send_stream_with_preheader_cancel( + client, req, disconnect_event, request = request + ) if resp is None: api_monitor.finish(monitor_id, "cancelled") return @@ -13222,6 +13794,9 @@ async def _responses_stream( yield _sse("response.completed", completed_response) async def admitted_event_generator(): + # Register for the body's whole lifetime, admission wait included: the run holds a decode + # slot from here on, so /load and /unload must count it. __exit__ runs from the finally below. + _tracker.__enter__() lease = reservation.lease_nowait() admission_wait_started_at = None stream_started = False @@ -13238,11 +13813,14 @@ async def _responses_stream( completion_id = resp_id, level = "debug", ) + # The tracked event, not just the client socket: registered above, so a forced swap's + # cancel_all() reaches this run while it is still queued. Otherwise it takes a lease it was + # told to give up and the post-cancel drain waits out the round trip it just cancelled. async for wait_item in _openai_admission_wait_stream_chunks( reservation, admission_config, request = request, - cancel_event = None, + cancel_event = cancel_event, ): if isinstance(wait_item, str): yield wait_item @@ -13263,7 +13841,7 @@ async def _responses_stream( await _raise_if_openai_admission_cancelled( reservation, request = request, - cancel_event = None, + cancel_event = cancel_event, ) iterator = event_generator() stream_started = True @@ -13312,6 +13890,7 @@ async def _responses_stream( if not stream_started: api_monitor.finish(monitor_id, "cancelled") reservation.cancel() + _tracker.__exit__(None, None, None) async def _responses_admission_unstarted_cleanup() -> None: api_monitor.finish(monitor_id, "cancelled") @@ -13909,6 +14488,24 @@ async def anthropic_messages( cancel_event, ) + async def _tracked_anthropic_non_streaming(coro): + """Register a non-streaming /v1/messages run with the swap gate. + + `stream` defaults to false, so this is the route's common shape, and all + three helpers hold llama-server for the whole await. /unload runs no idle + drain, so unregistered a swap tore the server down mid-request; only the + streaming siblings registered. No cancel keys, unlike the streaming + tool/plain siblings: the gate reaches a run through the registry, and + keys would add a cancel surface to a public API. + """ + _tracker = _TrackedCancel(cancel_event, model = model_name, kind = "messages") + _tracker.__enter__() + try: + return await _monitored_anthropic(coro) + finally: + # _monitored_anthropic's bookkeeping can throw; a leaked entry 409s later swaps. + _tracker.__exit__(None, None, None) + # ── Admission control ───────────────────────────────────── # Bound concurrent llama-server generations to the backend's serving slots via a # FIFO queue keyed by base_url (shared with /v1/chat/completions, same slots). @@ -14080,7 +14677,9 @@ async def anthropic_messages( request = request, cancel_event = cancel_event, ) - monitored = await _monitored_anthropic(coro) + # Registered only once admitted: a queued request is not holding + # llama-server, so it has no business blocking a swap. + monitored = await _tracked_anthropic_non_streaming(coro) return monitored except LlamaAdmissionTimeout as exc: coro.close() @@ -14151,6 +14750,8 @@ async def anthropic_messages( disable_parallel_tool_use = _disable_parallel, auto_heal_tool_calls = payload.auto_heal_tool_calls, nudge_tool_calls = payload.nudge_tool_calls, + request = request, + cancel_event = cancel_event, ) ) @@ -14326,134 +14927,132 @@ async def _anthropic_tool_stream( ) async def _stream(): - emitter = AnthropicStreamEmitter() - for line in emitter.start(message_id, model_name, input_tokens = input_tokens): - yield line - - captured_finish_reason = None - # Whether the response currently ends on a pending tool_use block (the - # client must act → stop_reason "tool_use") as opposed to final text. - # The server may run a tool and then keep generating, which flips this - # back to False — that is an end_turn (or max_tokens) response. - ends_on_tool_use = False - tool_blocks_emitted = 0 - drop_until_tool_end = False - # Last drop-branch keepalive, seeded to stream start so a chatty tool busy - # past the stall window still gets a keepalive though its events are dropped. - _last_drop_keepalive = time.monotonic() - - gen = run_gen() - _next_task = None - # Watcher to cancel on disconnect: the in-loop poll fires only between - # events, so a mid-prefill disconnect would otherwise hold the decode slot. - disconnect_watcher = asyncio.create_task( - _await_disconnect_then_cancel(request, cancel_event) - ) + # The server-tool loop decodes on llama-server for its whole body, so without an entry a + # non-forced /unload saw zero generations and tore the server down mid-response. Entered + # inside the body generator so a response whose body never starts leaves nothing behind. + # No thread_id: public API surface. + _tracker = _TrackedCancel(cancel_event, model = model_name, kind = "messages") + _tracker.__enter__() try: - while True: - if cancel_event.is_set() or await request.is_disconnected(): - cancel_event.set() - return - # Stall keepalive (see GGUF tool stream): silent backend segments - # must not leave the SSE stream idle past proxy timeouts. - _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) - while True: - _done_tasks, _ = await asyncio.wait( - {_next_task}, - timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, - ) - if _done_tasks: - break - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - event = _next_task.result() - # Done; drop the reference so the finally-block drain no-ops. - _next_task = None - if event is _sentinel: - break - etype = event.get("type") - if etype == "heartbeat": - # Tool-wrapper heartbeat -> SSE keepalive, checked BEFORE the drop - # skip: a dropped tool still runs server-side and its events keep the - # stall keepalive from firing, so dropping heartbeats would go silent. - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - continue - if etype in ("tool_output", "tool_args"): - # Live stdout / arg streaming have no Anthropic Messages equivalent - # (the full call/result follow in tool_use / tool_result), so drop them. - # They keep the stall keepalive from firing, so a chatty tool would go - # silent past the ~100s proxy cap; emit a rate-limited keepalive instead. - _now = time.monotonic() - if _now - _last_drop_keepalive >= _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S: - _last_drop_keepalive = _now - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - continue - if drop_until_tool_end: - # disable_parallel_tool_use: skip every event until (and - # including) this dropped tool call's tool_end. - if etype == "tool_end": - drop_until_tool_end = False - continue - if etype == "metadata": - _fr = event.get("finish_reason") - if _fr is not None: - captured_finish_reason = _fr - # Strip leaked tool-call XML from content events first, so a - # content event that was purely tool XML doesn't count as text. - # Protected helper preserves rehearsal and balanced - # [TOOL_CALLS] trailing prose (raw _TOOL_XML_RE.sub corrupts both). - if etype == "content": - event = dict(event) - event["text"] = _strip_tool_xml_for_display( - event["text"], - auto_heal_tool_calls = True, - enabled_tool_names = _display_names, - ) - # disable_parallel_tool_use: keep only the first tool_use block, - # dropping every later tool_start and its paired tool_end (robust - # to empty tool-call ids — tracked by state, not id matching). - if etype == "tool_start": - if disable_parallel_tool_use and tool_blocks_emitted >= 1: - drop_until_tool_end = True - continue - ends_on_tool_use = True - elif etype == "tool_end": - tool_blocks_emitted += 1 - # A tool_end means Unsloth executed the tool server-side, so - # the response no longer ends on a pending client action. - # Without this, a server tool that produces no trailing text - # would be mislabeled stop_reason "tool_use", telling the - # client to run a tool Unsloth already ran. - ends_on_tool_use = False - elif etype == "content" and event.get("text"): - ends_on_tool_use = False - for line in emitter.feed(event): - yield line - except Exception as e: - logger.error("anthropic_messages stream error: %s", e) - # force = True so an unclassified mid-stream failure (llama-server crash, - # decode OOM, dropped socket) still emits an SSE error and returns, instead - # of a normal message_stop that masks a truncated turn as a clean finish. - _error_event = _anthropic_stream_error_event(e, force = True) - if _error_event is not None: - yield _error_event - return - finally: - await _stop_local_disconnect_cancel_watcher(disconnect_watcher) - # Drain a still-running next(gen) worker before closing, so a mid-prefill - # disconnect releases the thread/generator/tool resources. Closing first - # would race into ValueError('generator already executing'). - await _drain_pending_next_task(_next_task, cancel_event) - if gen is not None: - try: - await asyncio.to_thread(gen.close) - except (RuntimeError, ValueError): - pass + emitter = AnthropicStreamEmitter() + for line in emitter.start(message_id, model_name, input_tokens = input_tokens): + yield line - stop_reason = openai_finish_to_anthropic_stop( - captured_finish_reason, had_tool_calls = ends_on_tool_use - ) - for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): - yield line + captured_finish_reason = None + # Response ends on a pending tool_use block rather than final text; a server tool + # that keeps generating flips this back to False. + ends_on_tool_use = False + tool_blocks_emitted = 0 + drop_until_tool_end = False + # Last drop-branch keepalive, seeded to stream start so a chatty tool busy past the + # stall window still gets one though its events are dropped. + _last_drop_keepalive = time.monotonic() + + gen = run_gen() + _next_task = None + # Watcher to cancel on disconnect: the in-loop poll fires only between events, + # so a mid-prefill disconnect would hold the decode slot. + disconnect_watcher = asyncio.create_task( + _await_disconnect_then_cancel(request, cancel_event) + ) + try: + while True: + if cancel_event.is_set() or await request.is_disconnected(): + cancel_event.set() + return + # Stall keepalive (see GGUF tool stream): silent backend segments must not + # leave the SSE stream idle past proxy timeouts. + _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) + while True: + _done_tasks, _ = await asyncio.wait( + {_next_task}, + timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, + ) + if _done_tasks: + break + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + event = _next_task.result() + # Done; drop the reference so the finally-block drain no-ops. + _next_task = None + if event is _sentinel: + break + etype = event.get("type") + if etype == "heartbeat": + # Tool-wrapper heartbeat -> SSE keepalive, checked BEFORE the drop skip: + # a dropped tool still runs and suppresses the stall keepalive. + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + continue + if etype in ("tool_output", "tool_args"): + # No Anthropic Messages equivalent (the full call/result follow in tool_use / + # tool_result), so drop them. They suppress the stall keepalive, so emit a + # rate-limited one instead of going silent past the ~100s proxy cap. + _now = time.monotonic() + if _now - _last_drop_keepalive >= _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S: + _last_drop_keepalive = _now + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + continue + if drop_until_tool_end: + # disable_parallel_tool_use: skip every event until (and + # including) this dropped tool call's tool_end. + if etype == "tool_end": + drop_until_tool_end = False + continue + if etype == "metadata": + _fr = event.get("finish_reason") + if _fr is not None: + captured_finish_reason = _fr + # Strip leaked tool-call XML first, so a purely-tool-XML content event doesn't + # count as text. The protected helper keeps rehearsal and balanced + # [TOOL_CALLS] trailing prose, which a raw sub corrupts. + if etype == "content": + event = dict(event) + event["text"] = _strip_tool_xml_for_display( + event["text"], + auto_heal_tool_calls = True, + enabled_tool_names = _display_names, + ) + # disable_parallel_tool_use: keep only the first tool_use block, dropping + # later tool_start/tool_end pairs (by state, not id: ids may be empty). + if etype == "tool_start": + if disable_parallel_tool_use and tool_blocks_emitted >= 1: + drop_until_tool_end = True + continue + ends_on_tool_use = True + elif etype == "tool_end": + tool_blocks_emitted += 1 + # Unsloth ran the tool server-side, so the response no longer ends on a pending + # client action; otherwise stop_reason "tool_use" tells the client to run it again. + ends_on_tool_use = False + elif etype == "content" and event.get("text"): + ends_on_tool_use = False + for line in emitter.feed(event): + yield line + except Exception as e: + logger.error("anthropic_messages stream error: %s", e) + # force = True so an unclassified mid-stream failure emits an SSE error instead + # of a message_stop that masks a truncated turn as a clean finish. + _error_event = _anthropic_stream_error_event(e, force = True) + if _error_event is not None: + yield _error_event + return + finally: + await _stop_local_disconnect_cancel_watcher(disconnect_watcher) + # Drain a still-running next(gen) worker first, so a mid-prefill disconnect releases + # its resources; closing first races into 'already executing'. + await _drain_pending_next_task(_next_task, cancel_event) + if gen is not None: + try: + await asyncio.to_thread(gen.close) + except (RuntimeError, ValueError): + pass + + stop_reason = openai_finish_to_anthropic_stop( + captured_finish_reason, had_tool_calls = ends_on_tool_use + ) + for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): + yield line + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(_stream()) @@ -14477,75 +15076,81 @@ async def _anthropic_plain_stream( input_tokens = await asyncio.to_thread(llama_backend.count_chat_tokens, openai_messages) async def _stream(): - emitter = AnthropicStreamEmitter() - for line in emitter.start(message_id, model_name, input_tokens = input_tokens): - yield line - - captured_finish_reason = None - - gen = run_gen() - _next_task = None - # Watcher to cancel on disconnect: the in-loop poll fires only between - # chunks, so a mid-prefill disconnect would otherwise hold the decode slot. - disconnect_watcher = asyncio.create_task( - _await_disconnect_then_cancel(request, cancel_event) - ) + # Registered like the tool stream above: this default /v1/messages path decodes on + # llama-server, so without an entry a non-forced /unload tore it down mid-response. + _tracker = _TrackedCancel(cancel_event, model = model_name, kind = "messages") + _tracker.__enter__() try: - while True: - if cancel_event.is_set() or await request.is_disconnected(): - cancel_event.set() - return - # Stall keepalive (see Anthropic tool stream) each window while - # next(gen) runs in a worker. - _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) - while True: - _done_tasks, _ = await asyncio.wait( - {_next_task}, - timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, - ) - if _done_tasks: - break - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - cumulative = _next_task.result() - # Done; drop the reference so the finally-block drain no-ops. - _next_task = None - if cumulative is _sentinel: - break - if isinstance(cumulative, dict): - if cumulative.get("type") == "metadata": - _fr = cumulative.get("finish_reason") - if _fr is not None: - captured_finish_reason = _fr - for line in emitter.feed(cumulative): - yield line - continue - # Plain generator yields cumulative text strings - for line in emitter.feed({"type": "content", "text": cumulative}): - yield line - except Exception as e: - logger.error("anthropic_messages stream error: %s", e) - # force = True so an unclassified mid-stream failure (llama-server crash, - # decode OOM, dropped socket) still emits an SSE error and returns, instead - # of a normal message_stop that masks a truncated turn as a clean finish. - _error_event = _anthropic_stream_error_event(e, force = True) - if _error_event is not None: - yield _error_event - return - finally: - await _stop_local_disconnect_cancel_watcher(disconnect_watcher) - # Drain a still-running next(gen) worker before closing, so a mid-prefill - # disconnect releases the thread/generator/model resources. Closing first - # would race into ValueError('generator already executing'). - await _drain_pending_next_task(_next_task, cancel_event) - if gen is not None: - try: - await asyncio.to_thread(gen.close) - except (RuntimeError, ValueError): - pass + emitter = AnthropicStreamEmitter() + for line in emitter.start(message_id, model_name, input_tokens = input_tokens): + yield line - stop_reason = openai_finish_to_anthropic_stop(captured_finish_reason, had_tool_calls = False) - for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): - yield line + captured_finish_reason = None + + gen = run_gen() + _next_task = None + # Watcher to cancel on disconnect: the in-loop poll fires only between chunks, + # so a mid-prefill disconnect would hold the decode slot. + disconnect_watcher = asyncio.create_task( + _await_disconnect_then_cancel(request, cancel_event) + ) + try: + while True: + if cancel_event.is_set() or await request.is_disconnected(): + cancel_event.set() + return + # Stall keepalive each window while next(gen) runs in a worker. + _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) + while True: + _done_tasks, _ = await asyncio.wait( + {_next_task}, + timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, + ) + if _done_tasks: + break + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + cumulative = _next_task.result() + # Done; drop the reference so the finally-block drain no-ops. + _next_task = None + if cumulative is _sentinel: + break + if isinstance(cumulative, dict): + if cumulative.get("type") == "metadata": + _fr = cumulative.get("finish_reason") + if _fr is not None: + captured_finish_reason = _fr + for line in emitter.feed(cumulative): + yield line + continue + # Plain generator yields cumulative text strings + for line in emitter.feed({"type": "content", "text": cumulative}): + yield line + except Exception as e: + logger.error("anthropic_messages stream error: %s", e) + # force = True so an unclassified mid-stream failure emits an SSE error instead + # of a message_stop that masks a truncated turn as a clean finish. + _error_event = _anthropic_stream_error_event(e, force = True) + if _error_event is not None: + yield _error_event + return + finally: + await _stop_local_disconnect_cancel_watcher(disconnect_watcher) + # Drain a still-running next(gen) worker first, so a mid-prefill disconnect releases + # its resources; closing first races into 'already executing'. + await _drain_pending_next_task(_next_task, cancel_event) + if gen is not None: + try: + await asyncio.to_thread(gen.close) + except (RuntimeError, ValueError): + pass + + stop_reason = openai_finish_to_anthropic_stop( + captured_finish_reason, had_tool_calls = False + ) + for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): + yield line + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(_stream()) @@ -14981,10 +15586,23 @@ async def _anthropic_passthrough_stream( # cancel_id mirrors the OpenAI passthrough so a per-run cancel POST # works without the caller having to know the local message_id. - _tracker = _TrackedCancel(cancel_event, cancel_id, session_id, message_id) - _tracker.__enter__() + # No thread_id: public API surface, but still registered so a reload cannot yank + # llama-server out from under it. Built here, entered below inside _stream(). + _tracker = _TrackedCancel( + cancel_event, + cancel_id, + session_id, + message_id, + model = model_name, + kind = "messages", + ) async def _stream(): + # Entered inside the body, not eagerly: aclose() runs no body on a generator + # that never started, so a client that drops first would leave the run + # registered until restart, 409-ing every swap. Ahead of the first yield, so + # the opening lines are covered as well. + _tracker.__enter__() emitter = AnthropicPassthroughEmitter() # Promote text-form tool calls (declared client tools only) into # tool_use blocks; verbatim behavior when healing is off or no tools. @@ -15162,8 +15780,16 @@ async def _anthropic_passthrough_non_streaming( disable_parallel_tool_use = False, auto_heal_tool_calls = None, nudge_tool_calls = None, + request: Optional[Request] = None, + cancel_event = None, ): - """Non-streaming client-side pass-through.""" + """Non-streaming client-side pass-through. + + Both POSTs run on a per-request client so a Stop or a forced swap can close + it and interrupt them. The pooled ``nonstreaming_client()`` cannot be closed + without disturbing unrelated calls, which left this path registered with the + swap gate but deaf to the event it registered. + """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_passthrough_payload( openai_messages, @@ -15181,138 +15807,162 @@ async def _anthropic_passthrough_non_streaming( backend_ctx = llama_backend.context_length, ) - try: - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _llama_non_streaming_generation_timeout(), - ) - except httpx.ConnectError as exc: - # Nothing was returned yet, so retry once against the respawned server's - # new port; the nudge retry below then reuses the same fresh URL. - retry_url = await _anthropic_passthrough_retry_url(llama_backend, exc) - if retry_url is None: - raise - target_url = retry_url - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _llama_non_streaming_generation_timeout(), + _client = _cancelable_nonstreaming_client() + _cancel_watcher = asyncio.create_task( + _await_cancel_or_disconnect_then_close_client( + cancel_event = cancel_event, + request = request, + client = _client, ) + ) - if resp.status_code != 200: - raise HTTPException( - status_code = resp.status_code, - detail = _friendly_upstream_error(resp.text[:500]), - ) - - data = resp.json() - # tool_choice arrives here already converted to the OpenAI shape. - _allowed_tools = heal_gate(auto_heal_tool_calls, openai_tools, tool_choice) - - # Opt-in single-retry nudge (mirrors the OpenAI passthrough): the model - # tried to call a tool but nothing usable came out; re-ask once with the - # prompt prefix intact so llama-server's KV cache is reused. - if ( - _allowed_tools - and nudge_enabled(nudge_tool_calls) - and nudge_should_retry(data, _allowed_tools, openai_tools) - ): - retry_body = { - **body, - "messages": [*body.get("messages", []), *nudge_messages(data, _allowed_tools)], - } + async def _post(payload_body): + nonlocal target_url try: - retry_resp = await nonstreaming_client().post( + return await _client.post( target_url, - json = retry_body, + json = payload_body, + timeout = _llama_non_streaming_generation_timeout(), + ) + except httpx.RequestError as exc: + # The watcher closes the client to break a blocked POST, so a transport error + # with the event set is the cancel, not a failure. + if cancel_event is not None and cancel_event.is_set(): + raise asyncio.CancelledError() + # Nothing was returned yet, so retry once against the respawned server's + # new port; the nudge retry below then reuses the same fresh URL. + retry_url = ( + await _anthropic_passthrough_retry_url(llama_backend, exc) + if isinstance(exc, httpx.ConnectError) + else None + ) + if retry_url is None: + raise + target_url = retry_url + return await _client.post( + target_url, + json = payload_body, timeout = _llama_non_streaming_generation_timeout(), ) - if retry_resp.status_code == 200: - retry_data = retry_resp.json() - if response_has_promotable_calls(retry_data, _allowed_tools, openai_tools): - data = retry_data - except (httpx.RequestError, ValueError) as exc: - logger.warning("tool-call nudge retry failed; keeping original: %s", exc) - choice = (data.get("choices") or [{}])[0] - message = choice.get("message") or {} - finish_reason = choice.get("finish_reason") + try: + resp = await _post(body) - healing_active = bool(_allowed_tools) - healed_events = ( - heal_openai_message_events(message, _allowed_tools, openai_tools) - if healing_active - else None - ) + if resp.status_code != 200: + raise HTTPException( + status_code = resp.status_code, + detail = _friendly_upstream_error(resp.text[:500]), + ) - content_blocks = [] - tool_calls = [] - if healed_events: - emitted_tool_uses = 0 - for kind, value in healed_events: - if kind == "text": - text = str(value).strip() + data = resp.json() + # tool_choice arrives here already converted to the OpenAI shape. + _allowed_tools = heal_gate(auto_heal_tool_calls, openai_tools, tool_choice) + + # Opt-in single-retry nudge (mirrors the OpenAI passthrough): the tool call came out + # unusable; re-ask with the prompt prefix intact so the KV cache is reused. + if ( + _allowed_tools + and nudge_enabled(nudge_tool_calls) + and nudge_should_retry(data, _allowed_tools, openai_tools) + ): + retry_body = { + **body, + "messages": [*body.get("messages", []), *nudge_messages(data, _allowed_tools)], + } + try: + retry_resp = await _post(retry_body) + if retry_resp.status_code == 200: + retry_data = retry_resp.json() + if response_has_promotable_calls(retry_data, _allowed_tools, openai_tools): + data = retry_data + except (httpx.RequestError, ValueError) as exc: + logger.warning("tool-call nudge retry failed; keeping original: %s", exc) + + choice = (data.get("choices") or [{}])[0] + message = choice.get("message") or {} + finish_reason = choice.get("finish_reason") + + healing_active = bool(_allowed_tools) + healed_events = ( + heal_openai_message_events(message, _allowed_tools, openai_tools) + if healing_active + else None + ) + + content_blocks = [] + tool_calls = [] + if healed_events: + emitted_tool_uses = 0 + for kind, value in healed_events: + if kind == "text": + text = str(value).strip() + if text: + content_blocks.append(AnthropicResponseTextBlock(text = text)) + continue + if disable_parallel_tool_use and emitted_tool_uses >= 1: + continue + fn = value.get("function") or {} + try: + args = json.loads(fn.get("arguments", "{}")) + except json.JSONDecodeError: + args = {} + tool_calls.append(value) + emitted_tool_uses += 1 + content_blocks.append( + AnthropicResponseToolUseBlock( + id = anthropic_tool_use_id(value.get("id")), + name = fn.get("name", ""), + input = args, + ) + ) + else: + text = message.get("content") or "" + if text: + # Keep unpromoted bytes when healing is active; legacy stripping is only for opted-out + # or no-client-tool requests. The protected helper preserves rehearsal and + # balanced [TOOL_CALLS] prose, gated on the declared tools so an inactive + # NAME[ARGS]{...} example is kept. + if not healing_active: + text = _strip_tool_xml_for_display( + text, + auto_heal_tool_calls = True, + enabled_tool_names = _display_tool_name_gate(openai_tools), + ) + text = text.strip() if text: content_blocks.append(AnthropicResponseTextBlock(text = text)) - continue - if disable_parallel_tool_use and emitted_tool_uses >= 1: - continue - fn = value.get("function") or {} - try: - args = json.loads(fn.get("arguments", "{}")) - except json.JSONDecodeError: - args = {} - tool_calls.append(value) - emitted_tool_uses += 1 - content_blocks.append( - AnthropicResponseToolUseBlock( - id = anthropic_tool_use_id(value.get("id")), - name = fn.get("name", ""), - input = args, - ) - ) - else: - text = message.get("content") or "" - if text: - # Keep unpromoted bytes when healing is active; legacy stripping is - # only for opted-out or no-client-tool requests. Protected helper (not - # raw _TOOL_XML_RE.sub): preserves rehearsal and balanced - # [TOOL_CALLS] trailing prose, gated on the declared tools so an - # inactive NAME[ARGS]{...} example in the final text is kept. - if not healing_active: - text = _strip_tool_xml_for_display( - text, - auto_heal_tool_calls = True, - enabled_tool_names = _display_tool_name_gate(openai_tools), - ) - text = text.strip() - if text: - content_blocks.append(AnthropicResponseTextBlock(text = text)) - tool_calls = message.get("tool_calls") or [] - if disable_parallel_tool_use and len(tool_calls) > 1: - tool_calls = tool_calls[:1] - for tc in tool_calls: - fn = tc.get("function") or {} - try: - args = json.loads(fn.get("arguments", "{}")) - except json.JSONDecodeError: - args = {} - content_blocks.append( - AnthropicResponseToolUseBlock( - id = anthropic_tool_use_id(tc.get("id")), - name = fn.get("name", ""), - input = args, + tool_calls = message.get("tool_calls") or [] + if disable_parallel_tool_use and len(tool_calls) > 1: + tool_calls = tool_calls[:1] + for tc in tool_calls: + fn = tc.get("function") or {} + try: + args = json.loads(fn.get("arguments", "{}")) + except json.JSONDecodeError: + args = {} + content_blocks.append( + AnthropicResponseToolUseBlock( + id = anthropic_tool_use_id(tc.get("id")), + name = fn.get("name", ""), + input = args, + ) ) - ) - stop_reason = openai_finish_to_anthropic_stop(finish_reason, had_tool_calls = bool(tool_calls)) + stop_reason = openai_finish_to_anthropic_stop( + finish_reason, had_tool_calls = bool(tool_calls) + ) - usage = data.get("usage") or {} - return _anthropic_message_json_response( - message_id, model_name, content_blocks, stop_reason, usage - ) + usage = data.get("usage") or {} + return _anthropic_message_json_response( + message_id, model_name, content_blocks, stop_reason, usage + ) + finally: + await _stop_local_disconnect_cancel_watcher(_cancel_watcher) + try: + await _client.aclose() + except Exception: + pass # ===================================================================== @@ -15694,7 +16344,7 @@ async def _openai_passthrough_stream( monitor_id: Optional[str] = None, ): _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() try: reservation, admission_config = _openai_llama_admission_reserve( diff --git a/studio/backend/run.py b/studio/backend/run.py index 5dfab9346a..8ef1ac06b8 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -1377,13 +1377,21 @@ def _apply_cli_tool_policy(enable_tools: "Optional[bool]") -> None: set_tool_policy(enable_tools) +# Mirror unsloth_cli/commands/studio.py's _PARALLEL_*: the admission queue caps concurrent +# chats at the slot count, so a direct launch matches the CLI (VRAM fit may still cut it +# back). Defined above run_server() so embedders that omit it do not serialise every chat. +_PARALLEL_MIN = 1 +_PARALLEL_MAX = 64 +_PARALLEL_DEFAULT_PLAIN = 4 + + def run_server( host: str = "127.0.0.1", port: int = 8888, frontend_path: Path = _DEFAULT_FRONTEND_PATH, silent: bool = False, api_only: bool = False, - llama_parallel_slots: int = 1, + llama_parallel_slots: int = _PARALLEL_DEFAULT_PLAIN, cloudflare: "Optional[bool]" = None, secure: bool = False, enable_tools: "Optional[bool]" = None, @@ -1399,7 +1407,8 @@ def run_server( frontend_path: Path to frontend build directory (optional) silent: Suppress startup messages api_only: API server only, no frontend (for Tauri desktop app) - llama_parallel_slots: parallel slots for llama-server + llama_parallel_slots: parallel slots for llama-server (default + _PARALLEL_DEFAULT_PLAIN, matching the CLI entry points) cloudflare: opt in to the public Cloudflare HTTPS tunnel for a wildcard bind. Tri-state: None (unset) and False both mean off; True enables it. --secure implies it (True) and rejects an explicit False. @@ -1817,13 +1826,6 @@ def run_server( return app -# Mirror unsloth_cli/commands/studio.py's _PARALLEL_*. Default 1 is for direct -# backend launches; `unsloth studio run` always passes its own value (4). -_PARALLEL_MIN = 1 -_PARALLEL_MAX = 64 -_PARALLEL_DEFAULT_PLAIN = 1 - - def _build_arg_parser(): """Build the backend CLI argument parser. @@ -1918,7 +1920,7 @@ def _build_arg_parser(): default = _PARALLEL_DEFAULT_PLAIN, help = ( f"llama-server parallel decode slots ({_PARALLEL_MIN}..{_PARALLEL_MAX}). " - f"Default {_PARALLEL_DEFAULT_PLAIN}; `unsloth studio run` uses 4." + f"Default {_PARALLEL_DEFAULT_PLAIN}." ), ) return parser diff --git a/studio/backend/state/active_generations.py b/studio/backend/state/active_generations.py new file mode 100644 index 0000000000..d1f2812c59 --- /dev/null +++ b/studio/backend/state/active_generations.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Registry of in-flight chat generations, keyed by conversation. + +New Chat leaves the previous conversation streaming, so /load and /unload need +to know which chats a reload would interrupt: they refuse with 409 unless the +caller opts in to cancelling them, and GET /inference/active-generations lets +the UI name them. A frontend guard alone would miss a second tab or a REST call. + +Entries hold the same threading.Event as the per-run cancel registry in +routes/inference.py, so cancel_all() closes each generation's own upstream +stream and never signals llama-server itself. + +A plain dict plus a threading.Lock: no signals, no process groups, no event loop +affinity, so it behaves identically on Linux, macOS, Windows and WSL. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from typing import Any, Optional + +# handle id -> entry. Keyed by handle, not thread_id: a tool continuation can register +# before the previous leg unregisters, and one key would drop the other. +_ACTIVE: dict[str, dict[str, Any]] = {} +_LOCK = threading.Lock() + + +class ActiveGeneration: + """Registers one in-flight generation for the duration of the block. + + Each __enter__ mints its own handle, so overlapping uses never clobber. + """ + + __slots__ = ("thread_id", "cancel_event", "model", "kind", "_handle") + + def __init__( + self, + cancel_event: threading.Event, + *, + thread_id: Optional[str] = None, + model: Optional[str] = None, + kind: str = "chat", + ): + self.thread_id = thread_id or None + self.cancel_event = cancel_event + self.model = model or None + self.kind = kind + self._handle: Optional[str] = None + + def __enter__(self) -> "ActiveGeneration": + self._handle = uuid.uuid4().hex + with _LOCK: + _ACTIVE[self._handle] = { + "handle": self._handle, + "thread_id": self.thread_id, + "model": self.model, + "kind": self.kind, + "started_at": time.time(), + "event": self.cancel_event, + } + return self + + def __exit__(self, *exc) -> bool: + handle, self._handle = self._handle, None + if handle is not None: + with _LOCK: + _ACTIVE.pop(handle, None) + return False + + +def snapshot() -> list[dict[str, Any]]: + """In-flight generations, newest last. Drops the Event: this is a response.""" + with _LOCK: + entries = list(_ACTIVE.values()) + entries.sort(key = lambda e: e["started_at"]) + return [ + { + "handle": e["handle"], + "thread_id": e["thread_id"], + "model": e["model"], + "kind": e["kind"], + "started_at": e["started_at"], + } + for e in entries + ] + + +def active_thread_ids() -> list[str]: + """Distinct conversation ids with a generation in flight, in start order. + + A first turn that races persistence has no thread id yet: count() sees it, + this cannot name it. + """ + seen: list[str] = [] + for e in snapshot(): + tid = e["thread_id"] + if tid and tid not in seen: + seen.append(tid) + return seen + + +def count() -> int: + """Number of generations currently in flight.""" + with _LOCK: + return len(_ACTIVE) + + +def cancel_all() -> int: + """Signal every in-flight generation to stop. Returns how many were signalled. + + Only sets the cancel events; each stream tears itself down. Entries are + removed by their own __exit__, so one mid-cleanup is neither lost nor double + counted. + """ + with _LOCK: + events = [e["event"] for e in _ACTIVE.values()] + for ev in events: + try: + ev.set() + except Exception: + pass + return len(events) + + +def cancel_thread(thread_id: str) -> int: + """Signal only the generations belonging to ``thread_id``.""" + if not thread_id: + return 0 + with _LOCK: + events = [e["event"] for e in _ACTIVE.values() if e["thread_id"] == thread_id] + for ev in events: + try: + ev.set() + except Exception: + pass + return len(events) + + +def reset_for_tests() -> None: + """Drop every entry. Test-only; never called from request paths.""" + with _LOCK: + _ACTIVE.clear() diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.py new file mode 100644 index 0000000000..aa087fe4ea --- /dev/null +++ b/studio/backend/tests/test_active_generations.py @@ -0,0 +1,2635 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Parallel chats: the active-generation registry and the model-swap gate. + +A load/unload has to know which streaming chats it would interrupt. Everything +under test is a dict + threading.Lock, so this passes on every platform. +""" + +import os +import sys +import threading + +import pytest + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +from state import active_generations + + +@pytest.fixture(autouse = True) +def _clean_registry(): + active_generations.reset_for_tests() + yield + active_generations.reset_for_tests() + + +# ── registry ────────────────────────────────────────────────────────── + + +def test_registry_starts_empty(): + assert active_generations.count() == 0 + assert active_generations.snapshot() == [] + assert active_generations.active_thread_ids() == [] + + +def test_entry_lives_only_for_the_block(): + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1", model = "m"): + assert active_generations.count() == 1 + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.count() == 0 + assert active_generations.active_thread_ids() == [] + + +def test_entry_is_removed_even_when_the_block_raises(): + ev = threading.Event() + with pytest.raises(RuntimeError): + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + raise RuntimeError("stream blew up") + assert active_generations.count() == 0 + + +def test_overlapping_runs_on_one_thread_both_register(): + # A tool continuation registers its next leg before the previous unwinds. + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t1"): + assert active_generations.count() == 2 + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.count() == 1 + assert active_generations.count() == 0 + + +def test_snapshot_is_json_safe_and_ordered_by_start(): + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "first", model = "m1"): + with active_generations.ActiveGeneration(b, thread_id = "second", model = "m2"): + snap = active_generations.snapshot() + assert [e["thread_id"] for e in snap] == ["first", "second"] + # The threading.Event must not leak into an HTTP response body. + assert all("event" not in e for e in snap) + assert {"handle", "thread_id", "model", "kind", "started_at"} == set(snap[0]) + + +def test_thread_ids_are_deduped_and_skip_unnamed_runs(): + a, b, c = threading.Event(), threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t1"): + # A brand-new chat whose first turn races persistence has no id yet. + with active_generations.ActiveGeneration(c, thread_id = None): + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.count() == 3 + + +# ── cancellation ────────────────────────────────────────────────────── + + +def test_cancel_all_sets_every_event(): + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + assert active_generations.cancel_all() == 2 + assert a.is_set() and b.is_set() + + +def test_cancel_all_on_an_empty_registry_is_a_no_op(): + assert active_generations.cancel_all() == 0 + + +def test_cancel_thread_leaves_siblings_alone(): + # Per-thread Stop: the rest keep generating, llama-server is untouched. + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + assert active_generations.cancel_thread("t1") == 1 + assert a.is_set() + assert not b.is_set() + + +def test_cancel_thread_with_no_match_is_a_no_op(): + a = threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + assert active_generations.cancel_thread("nope") == 0 + assert active_generations.cancel_thread("") == 0 + assert not a.is_set() + + +def test_cancel_does_not_unregister_entries(): + # __exit__ owns removal, so a generation mid-cleanup is not lost. + a = threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + active_generations.cancel_all() + assert active_generations.count() == 1 + + +# ── concurrency ─────────────────────────────────────────────────────── + + +def test_registry_survives_concurrent_register_unregister(): + errors: list[BaseException] = [] + barrier = threading.Barrier(8) + + def worker(i: int) -> None: + try: + barrier.wait(timeout = 10) + for _ in range(50): + with active_generations.ActiveGeneration(threading.Event(), thread_id = f"t{i}"): + active_generations.snapshot() + except BaseException as exc: # noqa: BLE001 - surfaced via assert below + errors.append(exc) + + threads = [threading.Thread(target = worker, args = (i,)) for i in range(8)] + for t in threads: + t.start() + for t in threads: + t.join(timeout = 30) + + assert errors == [] + assert active_generations.count() == 0 + + +# ── the model-swap gate ─────────────────────────────────────────────── + + +# The gate lives in routes.inference, which pulls the whole inference stack. +def _route_gate(): + pytest.importorskip("fastapi", reason = "inference stack not installed") + routes_inference = pytest.importorskip( + "routes.inference", reason = "inference stack not installed" + ) + return routes_inference._raise_or_cancel_active_generations + + +@pytest.fixture +def gate(): + return _route_gate() + + +def test_gate_allows_a_swap_when_nothing_is_generating(gate): + assert gate(force = False, action = "Loading a model") == 0 + + +def test_gate_refuses_with_409_and_names_the_chats(gate): + from fastapi import HTTPException + + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + with pytest.raises(HTTPException) as exc: + gate(force = False, action = "Loading a model") + assert exc.value.status_code == 409 + detail = exc.value.detail + assert detail["error"] == "active_generations" + assert detail["running"] == 2 + assert detail["thread_ids"] == ["t1", "t2"] + # Refusing must not cancel anything. + assert not a.is_set() and not b.is_set() + + +def test_gate_message_is_singular_for_one_chat(gate): + from fastapi import HTTPException + + with active_generations.ActiveGeneration(threading.Event(), thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + gate(force = False, action = "Unloading the model") + message = exc.value.detail["message"] + assert "1 chat that is still generating" in message + assert "Unloading the model" in message + + +def test_gate_force_cancels_and_returns_the_count(gate): + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + assert gate(force = True, action = "Loading a model") == 2 + assert a.is_set() and b.is_set() + + +def test_gate_force_with_nothing_running_is_a_no_op(gate): + assert gate(force = True, action = "Loading a model") == 0 + + +# ── the route wiring ────────────────────────────────────────────────── + + +def test_tracked_cancel_registers_the_thread_for_its_block(): + # The single place a generation is recorded, so every streaming path gets it. + _route_gate() + from routes.inference import _TrackedCancel + + ev = threading.Event() + tracker = _TrackedCancel(ev, "cancel-1", thread_id = "t1", model = "m") + tracker.__enter__() + try: + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.snapshot()[0]["model"] == "m" + finally: + tracker.__exit__(None, None, None) + assert active_generations.count() == 0 + + +def test_tracked_cancel_shares_its_event_with_the_registry(): + # Reusing the per-run event is what keeps a forced reload off llama-server. + _route_gate() + from routes.inference import _TrackedCancel + + ev = threading.Event() + tracker = _TrackedCancel(ev, "cancel-1", thread_id = "t1") + tracker.__enter__() + try: + active_generations.cancel_all() + assert ev.is_set() + finally: + tracker.__exit__(None, None, None) + + +def _stub_load_route(monkeypatch, *, active_model_name): + """Point POST /load at an in-memory safetensors backend. + + active_model_name == the requested path makes the request idempotent, so + _load_model_impl takes its already_loaded fast return. + """ + from types import SimpleNamespace + + import routes.inference as inf_mod + + monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", lambda: None) + monkeypatch.setattr(inf_mod, "validate_extra_args", lambda args: []) + monkeypatch.setattr( + inf_mod, + "resolve_effective_chat_template_override", + lambda model_identifier = None, user_override = None: None, + ) + monkeypatch.setattr(inf_mod, "load_inference_config", lambda name: {}) + monkeypatch.setattr( + inf_mod, + "_detect_safetensors_features", + lambda backend, template, tools = None: { + "supports_reasoning": False, + "reasoning_style": "enable_thinking", + "reasoning_effort_levels": [], + "reasoning_always_on": False, + "supports_preserve_thinking": False, + "supports_tools": False, + }, + ) + monkeypatch.setattr(inf_mod, "_resolve_loaded_trust_remote_code", lambda *a, **k: False) + monkeypatch.setattr( + inf_mod, + "get_inference_backend", + lambda: SimpleNamespace(active_model_name = active_model_name, models = {}), + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace(is_loaded = False, hf_variant = None, model_identifier = None), + ) + return inf_mod + + +def test_idempotent_load_neither_refuses_nor_cancels_running_chats(monkeypatch): + # Re-applying the resident model hits already_loaded: no llama-server touch, no 409, no stopped chats. + _route_gate() + import asyncio + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/A") + + for force in (False, True): + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = asyncio.run( + inf_mod.load_model( + LoadRequest(model_path = "org/A", force_cancel_active = force), + object(), + "tester", + ) + ) + assert response.status == "already_loaded" + assert not ev.is_set() + + +def test_a_real_reload_still_refuses_while_chats_stream(monkeypatch): + # A load that would really replace the model still 409s and names the chats. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run(inf_mod.load_model(LoadRequest(model_path = "org/A"), object(), "tester")) + assert exc.value.status_code == 409 + assert exc.value.detail["thread_ids"] == ["t1"] + assert not ev.is_set() + + +def test_a_forced_load_that_fails_preflight_leaves_the_chats_alone(monkeypatch): + # Preflight can still reject after the user confirms, so cancelling first ends chats for nothing. + _route_gate() + import asyncio + import contextlib + + from fastapi import HTTPException + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + # Stands in for any preflight refusal; a None here is the route's own 400. + monkeypatch.setattr(inf_mod.ModelConfig, "from_identifier", staticmethod(lambda **kwargs: None)) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.load_model( + LoadRequest(model_path = "org/A", force_cancel_active = True), + object(), + "tester", + ) + ) + # The load was rejected, so the chat must still be streaming. + assert not ev.is_set() + assert active_generations.count() == 1 + assert exc.value.status_code == 400 + + +def _stub_standard_load_route(monkeypatch): + """Drive _load_model_impl down the Unsloth path as far as the pre-teardown drain.""" + import contextlib + from types import SimpleNamespace + + import routes.inference as inf_mod + + real_sidecar_check = inf_mod._raise_if_sidecar_swap_in_progress + _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + # _stub_load_route neutralises the sidecar guard; this test is about it. + monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", real_sidecar_check) + monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + monkeypatch.setattr(inf_mod, "_mlx_distributed_launch_detected", lambda: False) + monkeypatch.setattr( + inf_mod.ModelConfig, + "from_identifier", + staticmethod( + lambda **kwargs: SimpleNamespace( + is_gguf = False, + identifier = "org/A", + display_name = "A", + is_vision = False, + gguf_hf_repo = None, + gguf_variant = None, + ) + ), + ) + monkeypatch.setattr(inf_mod, "_effective_load_in_4bit", lambda config, requested: False) + monkeypatch.setattr(inf_mod, "_resolve_inherited_extra_args", lambda *a, **k: None) + monkeypatch.setattr(inf_mod, "_guard_chat_load_against_training", lambda *a, **k: None) + return inf_mod + + +def test_a_sidecar_swap_reserved_during_the_drain_never_strands_cancelled_chats(monkeypatch): + # A sidecar install can reserve the swap window during the pre-teardown drain, so the recheck + # after it is the last rejection point and must precede the cancel, else chats die for nothing. + _route_gate() + import asyncio + import time + from types import SimpleNamespace + + from fastapi import HTTPException + + from core.inference import llama_keepwarm as kw + from models.inference import LoadRequest + + import utils.transformers_version as tv + + inf_mod = _stub_standard_load_route(monkeypatch) + reserved = {"v": False} + monkeypatch.setattr(tv, "sidecar_swap_in_progress", lambda: reserved["v"]) + + # Two tracked requests; the install reserves the window mid-drain when the uncancellable one ends. + monkeypatch.setattr(kw, "_inflight", 2) + + def _installer(): + time.sleep(0.10) + kw._inflight = 1 # the non-cancellable request finished ... + reserved["v"] = True # ... and an install reserved the swap window + time.sleep(0.35) + kw._inflight = 0 # the chat's own request drains last + + thread = threading.Thread(target = _installer, daemon = True) + ev = threading.Event() + try: + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + thread.start() + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.load_model( + LoadRequest(model_path = "org/A", force_cancel_active = True), + SimpleNamespace( + app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1)) + ), + "tester", + ) + ) + # Rejected, so the chat traded for a model it never got must still stream. + assert not ev.is_set() + assert active_generations.count() == 1 + assert exc.value.status_code == 409 + assert "transformers installation" in str(exc.value.detail) + finally: + thread.join(timeout = 5) + kw._inflight = 0 + + +def _stub_unload_backends(monkeypatch, *, llama, backend): + """Point the /unload route at in-memory backends.""" + import routes.inference as inf_mod + from core.inference import llama_keepwarm as kw + + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: llama) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "is_registered_native_path_label", lambda *a: False) + monkeypatch.setattr(kw, "note_model_unloaded", lambda: None) + return inf_mod, kw + + +def test_unload_rechecks_active_generations_under_the_lifecycle_gate(monkeypatch): + # Without the recheck, a chat that starts while this queues on the gate is torn down mid-stream. + _route_gate() + import asyncio + from types import SimpleNamespace + + from fastapi import HTTPException + + from models.inference import UnloadRequest + + torn_down: list[str] = [] + inf_mod, kw = _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = True, + model_identifier = "org/A-GGUF", + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: None, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + started = active_generations.ActiveGeneration(ev, thread_id = "t1") + + async def drive(): + # A load holds the lifecycle gate, so the unload queues behind it. + kw._lifecycle_lock.acquire() + task = asyncio.create_task( + inf_mod.unload_model(UnloadRequest(model_path = "org/A-GGUF"), "tester") + ) + entered = False + try: + await asyncio.sleep(0.1) # the route is polling the gate + started.__enter__() # a chat starts in the meantime + entered = True + finally: + kw._lifecycle_lock.release() + try: + return await asyncio.wait_for(task, timeout = 5) + finally: + if entered: + started.__exit__(None, None, None) + + with pytest.raises(HTTPException) as exc: + asyncio.run(drive()) + + # 409, not the catch-all 500 the route wraps unexpected failures in. + assert exc.value.status_code == 409 + assert exc.value.detail["error"] == "active_generations" + assert torn_down == [] + assert not ev.is_set() + + +def _run_unload( + inf_mod, + monkeypatch, + *, + loaded_gguf, + requested, + force, + torn_down, + unload_model = None, +): + """Drive POST /unload against a backend pair with ``loaded_gguf`` resident. + + ``unload_model`` overrides the GGUF teardown so a caller can observe what the + world looked like at the moment of teardown, not just afterwards. + """ + import asyncio + from types import SimpleNamespace + + from models.inference import UnloadRequest + + _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = True, + model_identifier = loaded_gguf, + unload_model = unload_model or (lambda: torn_down.append("gguf")), + ), + # Nothing on the standard backend: the GGUF above is what is resident. + backend = SimpleNamespace( + get_loading_model = lambda: None, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + return asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = requested, force_cancel_active = force), "tester" + ) + ) + + +def test_forced_unload_of_a_stale_model_path_leaves_the_chats_alone(monkeypatch): + # Eject naming a model another tab swapped out: a no-op success; cancelling first loses runs. + _route_gate() + import routes.inference as inf_mod + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/B-GGUF", # what the other tab actually loaded + requested = "org/A-GGUF", # this tab's stale idea of it + force = True, + torn_down = torn_down, + ) + assert not ev.is_set() + assert active_generations.count() == 1 + # The resident GGUF was never touched, so nothing was worth cancelling. + assert "gguf" not in torn_down + assert response.status == "unloaded" + + +def test_forced_unload_of_the_loaded_model_still_stops_its_chats(monkeypatch): + # A real unload must still cancel, or llama-server goes down mid-stream. + _route_gate() + import routes.inference as inf_mod + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + ) + assert ev.is_set() + assert torn_down == ["gguf"] + assert response.status == "unloaded" + + +def test_forced_unload_lets_the_cancelled_chats_unwind_before_teardown(monkeypatch): + # /unload used to tear down right after the cancel, so a stream told to stop but not yet + # finished lost its server. Assert the count hits zero BEFORE unload_model runs. + _route_gate() + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + inflight = {"n": 1} + seen = {} + + def _count(current_request_counted = True, *, include_pending = True): + # Unwinds one poll after the cancel, like a stream noticing its event. + if inflight["n"] > 0: + inflight["n"] -= 1 + return inflight["n"] + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0) + + torn_down: list[str] = [] + ev = threading.Event() + + def _record_teardown(): + seen["inflight_at_teardown"] = inflight["n"] + torn_down.append("gguf") + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + unload_model = _record_teardown, + ) + assert ev.is_set() + + assert torn_down == ["gguf"] + assert seen["inflight_at_teardown"] == 0 + assert response.status == "unloaded" + + +def test_unload_drains_on_the_middleware_count_not_just_the_registry(monkeypatch): + # A request past the middleware but not yet at its _TrackedCancel is counted but unregistered, so + # the drain reads the middleware count, not "did we cancel anything": one poll on a quiet server. + _route_gate() + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + polls = {"n": 0} + + def _count(current_request_counted = True, *, include_pending = True): + polls["n"] += 1 + return 0 + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + + torn_down: list[str] = [] + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + ) + assert torn_down == ["gguf"] + # Polled, but returned on the first read rather than waiting anything out. + assert polls["n"] == 1 + assert response.status == "unloaded" + + +def test_unforced_unload_of_a_stale_model_path_is_still_a_no_op(monkeypatch): + # Same stale Eject unforced: it reaches no teardown, so refusing strands the stale tab's selection. + _route_gate() + import routes.inference as inf_mod + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/B-GGUF", # what the other tab actually loaded + requested = "org/A-GGUF", # this tab's stale idea of it + force = False, + torn_down = torn_down, + ) + assert not ev.is_set() + assert active_generations.count() == 1 + # The resident GGUF was untouched; only the standard backend's stale-path no-op ran. + assert torn_down == ["unsloth"] + assert response.status == "unloaded" + + +def test_unforced_unload_of_the_loaded_model_still_refuses_while_chats_stream(monkeypatch): + # The stale skip above must not disarm the gate for a real replacement. + _route_gate() + import routes.inference as inf_mod + + from fastapi import HTTPException + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = False, + torn_down = torn_down, + ) + assert exc.value.status_code == 409 + assert exc.value.detail["thread_ids"] == ["t1"] + assert torn_down == [] + assert not ev.is_set() + + +def test_unforced_unload_still_refuses_while_a_gguf_load_is_in_flight(monkeypatch): + # A stale tab's Eject naming the PREVIOUS model while a different one loads. The GGUF branch + # evicts a live llama-server, so a chat on the previous model must get the 409, not be killed. + _route_gate() + import asyncio + from types import SimpleNamespace + + from fastapi import HTTPException + + from models.inference import UnloadRequest + + torn_down: list[str] = [] + inf_mod, _kw = _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = False, # spawned, health check not passed: mid-load + model_identifier = "org/B-GGUF", + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: None, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = "org/A-GGUF", force_cancel_active = False), + "tester", + ) + ) + assert exc.value.status_code == 409 + assert torn_down == [] + assert not ev.is_set() + + +def test_cancelling_an_in_flight_standard_load_is_not_refused_by_the_chat_gate(monkeypatch): + # The real cancelLoading shape: unforced /unload naming the still-LOADING model. It replaces + # nothing, so it cannot interrupt a chat and must not 409 (the frontend would drop the error). + _route_gate() + import asyncio + from types import SimpleNamespace + + from models.inference import UnloadRequest + + cancelled: list[str] = [] + torn_down: list[str] = [] + inf_mod, _kw = _stub_unload_backends( + monkeypatch, + # Nothing on llama-server: the load in flight is a safetensors one. + llama = SimpleNamespace( + is_active = False, + is_loaded = False, + model_identifier = None, + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: "org/B", + cancel_load = lambda path: bool(cancelled.append(path)) or True, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = "org/B", force_cancel_active = False), "tester" + ) + ) + # The chat on the previous model is untouched: the load never reached it. + assert not ev.is_set() + assert active_generations.count() == 1 + assert response.status == "unloaded" + assert cancelled == ["org/B"] + assert torn_down == [] + + +def test_cancelling_an_in_flight_gguf_load_is_not_refused_by_the_chat_gate(monkeypatch): + # Same cancelLoading shape on the GGUF fast path: killing that child ends a load, not a chat. + _route_gate() + import asyncio + from types import SimpleNamespace + + from models.inference import UnloadRequest + + torn_down: list[str] = [] + inf_mod, _kw = _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = False, # spawned, health check not passed: mid-load + model_identifier = "org/B-GGUF", + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: None, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = "org/B-GGUF", force_cancel_active = False), "tester" + ) + ) + assert not ev.is_set() + assert active_generations.count() == 1 + assert response.status == "unloaded" + assert torn_down == ["gguf"] + + +def _install_responses_stream_mock(monkeypatch, chunks): + """Point the direct /v1/responses GGUF pass-through at an in-process + llama-server. Mirrors the harness in test_responses_tool_passthrough.py.""" + import json + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + def handler(request): + content = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + content += "data: [DONE]\n\n" + return httpx.Response( + 200, + content = content.encode(), + headers = {"content-type": "text/event-stream"}, + ) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + is_vision = False, + context_length = 4096, + base_url = "http://llama.test", + supports_reasoning = True, + reasoning_always_on = False, + _request_reasoning_kwargs = ( + lambda enable_thinking = None, reasoning_effort = None, preserve_thinking = None: None + ), + ), + ) + return inf_mod + + +class _NeverDisconnectedRequest: + async def is_disconnected(self): + return False + + +def test_direct_responses_stream_is_visible_to_the_swap_gate(monkeypatch): + # /v1/responses streams straight to llama-server; unregistered, a non-forced /unload tore it down. + _route_gate() + import asyncio + + from models.inference import ChatMessage, ResponsesRequest + + inf_mod = _install_responses_stream_mock( + monkeypatch, [{"choices": [{"delta": {"content": "33"}}]}] + ) + payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF") + messages = [ChatMessage(role = "user", content = "hi")] + seen = {} + + async def run(): + response = await inf_mod._responses_stream(payload, messages, _NeverDisconnectedRequest()) + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + # And it unregisters, or one Codex call would 409 every later reload. + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_direct_responses_stream(monkeypatch): + # The registered event must be the one the stream watches, or a forced reload kills a live decode. + _route_gate() + import asyncio + + from models.inference import ChatMessage, ResponsesRequest + + inf_mod = _install_responses_stream_mock( + monkeypatch, + [ + {"choices": [{"delta": {"content": "3"}}]}, + {"choices": [{"delta": {"content": "3"}}]}, + ], + ) + payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF") + messages = [ChatMessage(role = "user", content = "hi")] + + async def run(): + response = await inf_mod._responses_stream(payload, messages, _NeverDisconnectedRequest()) + iterator = response.body_iterator + chunks = [await iterator.__anext__()] + assert active_generations.cancel_all() == 1 + async for chunk in iterator: + chunks.append(chunk) + return "".join(c.decode() if isinstance(c, bytes) else c for c in chunks) + + body = asyncio.run(run()) + + # Cancelled mid-stream: the run ends without a completed envelope. + assert "response.completed" not in body + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_responses_stream_still_queued_for_a_slot(monkeypatch): + # The run registers before it holds a decode slot, so cancel_all() must reach it while queued in + # admission; watching only the client socket lets it open a generation the swap already revoked. + _route_gate() + import asyncio + + from core.inference import llama_admission + from models.inference import ChatMessage, ResponsesRequest + + for name in ( + llama_admission.ADMISSION_CONTROL_ENV, + llama_admission.ADMISSION_QUEUE_TIMEOUT_ENV, + llama_admission.ADMISSION_KEEPALIVE_INTERVAL_ENV, + llama_admission.ADMISSION_MAX_QUEUE_ENV, + ): + monkeypatch.delenv(name, raising = False) + + inf_mod = _install_responses_stream_mock( + monkeypatch, [{"choices": [{"delta": {"content": "33"}}]}] + ) + payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF") + messages = [ChatMessage(role = "user", content = "hi")] + + llama_admission.reset_llama_admission_queues() + try: + + async def run(): + # Hold the backend's only decode slot so the run below has to queue. + queue = llama_admission.get_llama_admission_queue("http://llama.test") + holder = queue.reserve(capacity = 1, config = llama_admission.LlamaAdmissionConfig()) + assert holder.lease_nowait() is not None + response = await inf_mod._responses_stream( + payload, messages, _NeverDisconnectedRequest() + ) + chunks = [] + + async def drain(): + async for chunk in response.body_iterator: + chunks.append(chunk) + + task = asyncio.create_task(drain()) + for _ in range(500): + if active_generations.count() == 1: + break + await asyncio.sleep(0.01) + assert active_generations.count() == 1, "the queued run never registered" + assert active_generations.cancel_all() == 1 + # Unbounded queue by default: without the tracked event this never returns while the slot is held. + await asyncio.wait_for(task, timeout = 5) + return chunks + + chunks = asyncio.run(run()) + finally: + llama_admission.reset_llama_admission_queues() + + body = "".join(c.decode() if isinstance(c, bytes) else c for c in chunks) + # It gave up its place instead of taking the slot: no upstream call, no envelope. + assert "response.created" not in body + assert active_generations.count() == 0 + + +def _install_completions_stream_mock(monkeypatch, events): + """Point the /v1/completions proxy at an in-process llama-server.""" + import json + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + def handler(request): + # One network chunk per SSE event: the relay polls its cancel flag between upstream chunks. + async def _chunks(): + for event in events: + yield f"data: {json.dumps(event)}\n\n".encode() + yield b"data: [DONE]\n\n" + + return httpx.Response( + 200, + content = _chunks(), + headers = {"content-type": "text/event-stream"}, + ) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + context_length = 4096, + base_url = "http://llama.test", + model_identifier = "org/M-GGUF", + ), + ) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + + async def _no_auto_switch(request, current_subject): + return await request.json() + + monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch) + return inf_mod + + +class _CompletionsRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/completions reads.""" + + def __init__(self, body): + from types import SimpleNamespace + + self._body = body + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/completions") + + async def json(self): + return self._body + + +def test_completions_proxy_stream_is_visible_to_the_swap_gate(monkeypatch): + # /v1/completions relays from llama-server with no idle drain; unregistered, /unload tore it down. + _route_gate() + import asyncio + + inf_mod = _install_completions_stream_mock(monkeypatch, [{"choices": [{"text": "33"}]}]) + request = _CompletionsRequest( + {"prompt": "hi", "stream": True, "model": "org/M-GGUF", "max_tokens": 8} + ) + seen = {} + + async def run(): + response = await inf_mod.openai_completions(request, "tester") + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + # And it unregisters, or one completion would 409 every later reload. + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_completions_proxy_stream(monkeypatch): + # The registered event must be the one the relay watches, or a forced reload kills a live decode. + _route_gate() + import asyncio + + inf_mod = _install_completions_stream_mock( + monkeypatch, + [{"choices": [{"text": "3"}]}, {"choices": [{"text": "3"}]}], + ) + request = _CompletionsRequest( + {"prompt": "hi", "stream": True, "model": "org/M-GGUF", "max_tokens": 8} + ) + + async def run(): + response = await inf_mod.openai_completions(request, "tester") + iterator = response.body_iterator + chunks = [await iterator.__anext__()] + assert active_generations.cancel_all() == 1 + async for chunk in iterator: + chunks.append(chunk) + return b"".join(c if isinstance(c, bytes) else c.encode() for c in chunks) + + body = asyncio.run(run()) + + # Stopped after the first event instead of relaying the rest. + assert body.count(b'"text"') == 1 + assert active_generations.count() == 0 + + +def test_completions_proxy_non_stream_is_visible_to_the_swap_gate(monkeypatch): + # ``stream`` defaults to false, so the non-streaming branch is the common shape and holds + # llama-server throughout: unregistered, /unload counts zero and force_cancel_active has no event. + _route_gate() + import asyncio + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + seen = {} + + def handler(request): + # Sampled mid-flight: exactly the window a concurrent /unload would tear down in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + # And the gate must reach this run, not just see it. + seen["cancelled"] = active_generations.cancel_all() + return httpx.Response(200, json = {"id": "cmpl-x", "choices": [{"text": "33"}]}) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + # The pooled client too, so a route that took no per-request one still reaches this transport. + monkeypatch.setattr( + inf_mod, "nonstreaming_client", lambda: real_async_client(transport = transport) + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + context_length = 4096, + base_url = "http://llama.test", + model_identifier = "org/M-GGUF", + ), + ) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + + async def _no_auto_switch(request, current_subject): + return await request.json() + + monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch) + + request = _CompletionsRequest({"prompt": "hi", "model": "org/M-GGUF", "max_tokens": 8}) + + with pytest.raises(asyncio.CancelledError): + asyncio.run(inf_mod.openai_completions(request, "tester")) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # And it unregisters, or one completion would 409 every later reload. + assert active_generations.count() == 0 + + +class _EmbeddingsRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/embeddings reads.""" + + def __init__(self, body): + from types import SimpleNamespace + + self._body = body + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/embeddings") + self.state = SimpleNamespace(skip_api_monitor = True) + + async def json(self): + return self._body + + +def test_embeddings_proxy_is_visible_to_the_swap_gate(monkeypatch): + # /v1/embeddings holds llama-server for its whole HTTP call: unregistered, a non-forced /unload + # counts zero and kills the server mid-request (only /load waits on the middleware count). + _route_gate() + import asyncio + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + seen = {} + + def handler(request): + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + return httpx.Response(200, json = {"data": [{"embedding": [0.1, 0.2]}]}) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + monkeypatch.setattr( + inf_mod, "nonstreaming_client", lambda: real_async_client(transport = transport) + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + context_length = 4096, + base_url = "http://llama.test", + model_identifier = "org/M-GGUF", + ), + ) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + + async def _no_auto_switch(request, current_subject): + return await request.json() + + monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch) + + request = _EmbeddingsRequest({"input": "hi", "model": "org/M-GGUF"}) + + with pytest.raises(asyncio.CancelledError): + asyncio.run(inf_mod.openai_embeddings(request, "tester")) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # And it unregisters, or one embedding would 409 every later reload. + assert active_generations.count() == 0 + + +def test_active_generations_redacts_native_model_paths(monkeypatch): + # The legacy stream records active_model_name verbatim (an absolute path locally) and is the only + # place that serialises it: redact like the error paths so a remote client cannot learn host paths. + _route_gate() + import asyncio + import threading + from types import SimpleNamespace + + import routes.inference as inf_mod + from utils.native_path_leases import _remember_native_path_for_redaction + + secret_path = "/home/somebody/models/private-model.gguf" + _remember_native_path_for_redaction(secret_path, "private-model.gguf") + + request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 4))) + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: SimpleNamespace()) + + with active_generations.ActiveGeneration(threading.Event(), thread_id = "t1", model = secret_path): + body = asyncio.run(inf_mod.get_active_generations(request, "tester")) + + assert body["count"] == 1 + assert secret_path not in str(body) + assert body["active"][0]["model"] == "" + + +def test_legacy_generate_stream_is_visible_to_the_swap_gate(monkeypatch): + # The legacy /generate/stream decodes on the standard backend throughout: unregistered it passed + # the advertised 409 gate then blocked on the generation lock, and a forced swap had no event. + _route_gate() + import asyncio + from types import SimpleNamespace + + import routes.inference as inf_mod + from models.inference import GenerateRequest + + seen = {} + + def _fake_generate_chat_response(**kwargs): + # Sampled mid-generation: exactly the window an /unload would land in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + yield "hello" + yield "world" + + backend = SimpleNamespace( + active_model_name = "org/M", + models = {"org/M": {}}, + generate_chat_response = lambda **kw: _fake_generate_chat_response(**kw), + reset_generation_state = lambda *a: None, + resize_image = lambda img: img, + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend) + + async def _drain(): + response = await inf_mod.generate_stream( + GenerateRequest(messages = [{"role": "user", "content": "hi"}]), + _NeverDisconnectedRequest(), + current_subject = "tester", + ) + async for _ in response.body_iterator: + pass + + asyncio.run(_drain()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M" + assert seen["cancelled"] == 1 + # And it unregisters, or one legacy stream would 409 every later reload. + assert active_generations.count() == 0 + + +def _anthropic_stream_args(chunks): + """(request, cancel_event, run_gen) for the local Anthropic stream helpers.""" + cancel_event = threading.Event() + + def run_gen(): + def _gen(): + for chunk in chunks: + if cancel_event.is_set(): + return + yield chunk + + return _gen() + + return _NeverDisconnectedRequest(), cancel_event, run_gen + + +def test_local_anthropic_plain_stream_is_visible_to_the_swap_gate(monkeypatch): + # Only the client-tool pass-through registered, so the no-tool /v1/messages path died mid-response. + _route_gate() + import asyncio + + import routes.inference as inf_mod + + request, cancel_event, run_gen = _anthropic_stream_args(["3", "33"]) + seen = {} + + async def run(): + response = await inf_mod._anthropic_plain_stream( + request, cancel_event, run_gen, "msg_1", "org/M-GGUF" + ) + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_local_anthropic_plain_stream(monkeypatch): + # The event registered has to be the one the decode loop watches. + _route_gate() + import asyncio + + import routes.inference as inf_mod + + request, cancel_event, run_gen = _anthropic_stream_args(["3", "33", "333"]) + + async def run(): + response = await inf_mod._anthropic_plain_stream( + request, cancel_event, run_gen, "msg_1", "org/M-GGUF" + ) + iterator = response.body_iterator + chunks = [await iterator.__anext__()] + assert active_generations.cancel_all() == 1 + async for chunk in iterator: + chunks.append(chunk) + return "".join(c.decode() if isinstance(c, bytes) else c for c in chunks) + + body = asyncio.run(run()) + + assert cancel_event.is_set() + # Cancelled mid-stream: no clean message_stop envelope. + assert "message_stop" not in body + assert active_generations.count() == 0 + + +def test_local_anthropic_tool_stream_is_visible_to_the_swap_gate(monkeypatch): + # Same gap on the server-tool path (enable_tools / Anthropic server tools). + _route_gate() + import asyncio + + import routes.inference as inf_mod + + request, cancel_event, run_gen = _anthropic_stream_args( + [{"type": "content", "text": "3"}, {"type": "content", "text": "33"}] + ) + seen = {} + + async def run(): + response = await inf_mod._anthropic_tool_stream( + request, cancel_event, run_gen, "msg_1", "org/M-GGUF" + ) + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert active_generations.count() == 0 + + +def test_load_and_unload_requests_default_to_not_cancelling(): + pytest.importorskip("pydantic", reason = "pydantic not installed") + from models.inference import LoadRequest, UnloadRequest + + assert LoadRequest(model_path = "m").force_cancel_active is False + assert UnloadRequest(model_path = "m").force_cancel_active is False + assert LoadRequest(model_path = "m", force_cancel_active = True).force_cancel_active is True + + +def _parallel_constants(path: str) -> dict: + """Read the _PARALLEL_* constants from a file's source. + + Importing run.py would drag in the whole server to read three integers. + """ + import ast + + with open(path, encoding = "utf-8") as f: + tree = ast.parse(f.read()) + found = {} + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + name = getattr(target, "id", "") + if name.startswith("_PARALLEL_") and isinstance(node.value, ast.Constant): + found[name] = node.value.value + return found + + +def test_studio_defaults_to_more_than_one_decode_slot(): + # With one slot the admission queue serialises every chat. + consts = _parallel_constants(os.path.join(_backend, "run.py")) + + assert consts["_PARALLEL_DEFAULT_PLAIN"] > 1 + assert consts["_PARALLEL_MIN"] <= consts["_PARALLEL_DEFAULT_PLAIN"] <= consts["_PARALLEL_MAX"] + + +def test_cli_and_backend_parallel_defaults_agree(): + # argparse and the typer CLI are separate entry points into the same server. + backend = _parallel_constants(os.path.join(_backend, "run.py")) + cli_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(_backend))), + "unsloth_cli", + "commands", + "studio.py", + ) + cli = _parallel_constants(cli_path) + + assert cli["_PARALLEL_DEFAULT_PLAIN"] == backend["_PARALLEL_DEFAULT_PLAIN"] + + +def _run_server_parallel_default(path: str, consts: dict): + """Resolve run_server()'s llama_parallel_slots default from run.py's source.""" + import ast + + with open(path, encoding = "utf-8") as f: + tree = ast.parse(f.read()) + for node in tree.body: + if not isinstance(node, ast.FunctionDef) or node.name != "run_server": + continue + args = node.args.args + defaults = node.args.defaults + # defaults align with the tail of the positional arg list. + for arg, default in zip(args[len(args) - len(defaults) :], defaults): + if arg.arg != "llama_parallel_slots": + continue + if isinstance(default, ast.Constant): + return default.value + if isinstance(default, ast.Name): + return consts.get(default.id) + return None + return None + + +def test_run_server_default_matches_the_cli_parallel_default(): + # colab.py omits llama_parallel_slots, so the signature default is what Colab runs with. + run_path = os.path.join(_backend, "run.py") + consts = _parallel_constants(run_path) + + default = _run_server_parallel_default(run_path, consts) + + assert default is not None, "run_server() must keep a llama_parallel_slots default" + assert default == consts["_PARALLEL_DEFAULT_PLAIN"] + assert default > 1 + + +def test_colab_launcher_inherits_the_parallel_default(): + # Guard the inheritance itself: an explicit 1 here would resurrect the bug. + import ast + + colab_path = os.path.join(_backend, "colab.py") + with open(colab_path, encoding = "utf-8") as f: + tree = ast.parse(f.read()) + consts = _parallel_constants(os.path.join(_backend, "run.py")) + + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "run_server" + ] + assert calls, "colab.py must still launch the backend through run_server()" + for call in calls: + for kw in call.keywords: + if kw.arg != "llama_parallel_slots": + continue + value = kw.value.value if isinstance(kw.value, ast.Constant) else None + assert ( + value is None or value > 1 + ), "colab.py pins llama_parallel_slots to 1; Colab chats would serialise" + # Whether pinned or inherited, Colab must end up with more than one slot. + assert consts["_PARALLEL_DEFAULT_PLAIN"] > 1 + + +# ── the point of no return ──────────────────────────────────────────── + + +def test_a_forced_load_that_loses_to_a_sidecar_install_leaves_the_chats_alone(monkeypatch): + # The destructive cancel is the point of no return: nothing after it may reject the load. A sidecar + # install can reserve the window during preflight, so its recheck must run before, not after. + _route_gate() + import asyncio + import contextlib + from types import SimpleNamespace + + from fastapi import HTTPException + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + monkeypatch.setattr( + inf_mod.ModelConfig, + "from_identifier", + staticmethod( + lambda **kwargs: SimpleNamespace( + is_gguf = False, + identifier = "org/A", + display_name = "A", + is_vision = False, + is_lora = False, + path = None, + ) + ), + ) + monkeypatch.setattr(inf_mod, "_mlx_distributed_launch_detected", lambda: False) + monkeypatch.setattr(inf_mod, "_guard_chat_load_against_training", lambda *a, **k: None) + monkeypatch.setattr(inf_mod, "_resolve_inherited_extra_args", lambda *a, **k: None) + + # The two route-level checks pass, every check after them 409s. + seen = {"calls": 0} + + def _sidecar_reserved_during_preflight(): + seen["calls"] += 1 + if seen["calls"] > 2: + raise HTTPException( + status_code = 409, + detail = "A transformers installation is in progress. Retry when it completes.", + ) + + monkeypatch.setattr( + inf_mod, "_raise_if_sidecar_swap_in_progress", _sidecar_reserved_during_preflight + ) + + fastapi_request = SimpleNamespace( + app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1)) + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.load_model( + LoadRequest( + model_path = "org/A", + load_in_4bit = False, + force_cancel_active = True, + ), + fastapi_request, + "tester", + ) + ) + # The load was rejected, so the chat must still be streaming. + assert not ev.is_set() + assert active_generations.count() == 1 + assert exc.value.status_code == 409 + + +def test_anthropic_passthrough_registers_nothing_until_its_body_starts(): + # A pass-through response whose body never starts must leave both registries clean: a never-started + # async generator runs no body code (PEP 342), so an eagerly entered tracker never unregisters. + _route_gate() + import asyncio + import inspect + from types import SimpleNamespace + + from starlette.requests import ClientDisconnect + + import routes.inference as inf_mod + + llama_backend = SimpleNamespace( + base_url = "http://127.0.0.1:8080", + context_length = 4096, + count_chat_tokens = lambda messages, _unused, tools: 7, + ) + + async def _build(): + return await inf_mod._anthropic_passthrough_stream( + SimpleNamespace(), + threading.Event(), + llama_backend, + [{"role": "user", "content": "hi"}], + [], + 0.7, + 0.9, + 40, + 128, + "msg_1", + "org/A", + session_id = "s1", + cancel_id = "c1", + ) + + # Built and abandoned, as when the request task is cancelled before Starlette calls the response. + asyncio.run(_build()) + assert active_generations.count() == 0 + assert not inf_mod._CANCEL_REGISTRY + + # The client is gone at header time, so the first send fails and the body generator never runs. + async def _drive(): + response = await _build() + + async def _receive(): + return {"type": "http.disconnect"} + + async def _send(message): + raise OSError("client disconnected") + + with pytest.raises(ClientDisconnect): + await response({"type": "http"}, _receive, _send) + + asyncio.run(_drive()) + assert active_generations.count() == 0 + assert not inf_mod._CANCEL_REGISTRY + + # Still tracked once the body runs: the enter stays inside the generator, under the finally. + src = inspect.getsource(inf_mod._anthropic_passthrough_stream) + assert src.index("async def _stream()") < src.index("_tracker.__enter__()") + assert src.index("_tracker.__enter__()") < src.index("_tracker.__exit__(None, None, None)") + + +def test_audio_generation_is_visible_to_the_swap_gate(monkeypatch): + # /audio/generate is non-streaming and holds the model for the whole request: unregistered, a + # non-forced swap counted zero and could tear it down mid-TTS, and a forced one had no entry. + _route_gate() + import asyncio + from types import SimpleNamespace + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + seen = {} + + class _TtsBackend: + active_model_name = "org/TTS" + models = {"org/TTS": {"is_audio": True}} + + def generate_audio_response(self, **kwargs): + # Sampled mid-generation: the window a concurrent swap would tear down in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + return (b"RIFFfake", 24000) + + # is_loaded False picks the transformers TTS branch, not the GGUF one. + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace(is_loaded = False, _is_audio = False), + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: _TtsBackend()) + + async def _no_auto_switch(*a, **k): + return None + + monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch) + + payload = ChatCompletionRequest( + model = "org/TTS", + messages = [{"role": "user", "content": "hi"}], + thread_id = "thread-tts", + ) + asyncio.run(inf_mod.generate_audio(payload, request = None, current_subject = "tester")) + + assert seen["count"] == 1 + # Named, so the swap dialog can say which chat it would interrupt. + assert seen["snapshot"][0]["thread_id"] == "thread-tts" + # And it unregisters, or one TTS call would 409 every later reload. + assert active_generations.count() == 0 + + +class _ChatRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/chat/completions reads.""" + + def __init__(self): + from types import SimpleNamespace + + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/chat/completions") + self.state = SimpleNamespace(skip_api_monitor = True) + self.scope: dict = {} + + +def _standard_chat_stubs(monkeypatch, backend): + """Point /v1/chat/completions at a standard (non-GGUF) backend. + + ``supports_tools`` False keeps the request off the safetensors server-tool + loop, which registers on its own, so the plain default branch is exercised. + """ + from types import SimpleNamespace + + import routes.inference as inf_mod + + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = False, + supports_tools = False, + is_vision = False, + context_length = None, + ), + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + monkeypatch.setattr( + inf_mod, "_detect_safetensors_features", lambda *a, **k: {"supports_tools": False} + ) + + async def _no_auto_switch(*a, **k): + return None + + monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch) + return inf_mod + + +def test_standard_non_stream_chat_is_visible_to_the_swap_gate(monkeypatch): + # ``stream`` defaults to false, so this is the default shape of a standard chat and it holds the + # worker throughout. Only the streaming branch registered, so a swap truncated the completion. + _route_gate() + import asyncio + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + seen = {} + + class _StandardBackend: + active_model_name = "org/M" + models = {"org/M": {"chat_template_info": {"template": "chatml"}}} + + def generate_chat_response( + self, + *, + cancel_event = None, + stats_holder = None, + **kwargs, + ): + # Sampled mid-generation: exactly the window an /unload lands in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + # And the gate must reach this run, on the event the decode watches. + seen["cancelled"] = active_generations.cancel_all() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield "33" + + def reset_generation_state(self, caller_cancel_event = None): + pass + + _standard_chat_stubs(monkeypatch, _StandardBackend()) + + payload = ChatCompletionRequest( + model = "org/M", + messages = [{"role": "user", "content": "hi"}], + thread_id = "thread-chat", + ) + response = asyncio.run( + inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + # Named, so the swap dialog can say which chat it would interrupt. + assert seen["snapshot"][0]["thread_id"] == "thread-chat" + assert seen["cancelled"] == 1 + assert seen["reached_the_decode"] + # And it unregisters, or one completion would 409 every later reload. + assert active_generations.count() == 0 + + +def test_standard_non_stream_chat_unregisters_when_it_fails(monkeypatch): + # A raising backend must not strand an entry: that would 409 every later swap. + _route_gate() + import asyncio + + from fastapi import HTTPException + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + class _BrokenBackend: + active_model_name = "org/M" + models = {"org/M": {"chat_template_info": {"template": "chatml"}}} + + def generate_chat_response(self, **kwargs): + raise RuntimeError("decode exploded") + yield # pragma: no cover - generator marker + + def reset_generation_state(self, caller_cancel_event = None): + pass + + _standard_chat_stubs(monkeypatch, _BrokenBackend()) + + payload = ChatCompletionRequest(model = "org/M", messages = [{"role": "user", "content": "hi"}]) + with pytest.raises(HTTPException): + asyncio.run( + inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester") + ) + + assert active_generations.count() == 0 + + +def test_audio_input_non_stream_chat_is_visible_to_the_swap_gate(monkeypatch): + # An audio-input model with the default stream=false holds the standard worker throughout. Only + # the streaming sibling registered, so a non-forced swap could unload it mid-transcription. + _route_gate() + import asyncio + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + seen = {} + + class _AudioInputBackend: + active_model_name = "org/AUDIO-IN" + models = {"org/AUDIO-IN": {"has_audio_input": True}} + + def generate_audio_input_response( + self, + *, + cancel_event = None, + **kwargs, + ): + # Sampled mid-transcription: the window a concurrent swap lands in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield "33" + + def reset_generation_state(self, caller_cancel_event = None): + pass + + _standard_chat_stubs(monkeypatch, _AudioInputBackend()) + monkeypatch.setattr(inf_mod, "_decode_audio_base64", lambda _b64: object()) + + payload = ChatCompletionRequest( + model = "org/AUDIO-IN", + messages = [{"role": "user", "content": "transcribe this"}], + audio_base64 = "ZmFrZQ==", + thread_id = "thread-audio-in", + ) + response = asyncio.run( + inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + assert seen["snapshot"][0]["thread_id"] == "thread-audio-in" + assert seen["cancelled"] == 1 + assert seen["reached_the_decode"] + # And it unregisters, or one transcription would 409 every later reload. + assert active_generations.count() == 0 + + +def _anthropic_route_stubs(monkeypatch, **overrides): + """Minimal GGUF backend + request stub for the /v1/messages route.""" + from types import SimpleNamespace + + import routes.inference as inf_mod + from state.tool_policy import reset_tool_policy + + reset_tool_policy() + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + supports_tool_passthrough = True, + model_identifier = "org/M-GGUF", + base_url = "http://llama.test", + context_length = 4096, + count_chat_tokens = lambda *a, **k: 2, + ) + backend.__dict__.update(overrides) + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + return inf_mod + + +class _MessagesRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/messages reads.""" + + def __init__(self): + from types import SimpleNamespace + + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/messages") + self.state = SimpleNamespace(skip_api_monitor = True) + + +@pytest.mark.parametrize("with_server_tools", [False, True]) +def test_local_anthropic_non_stream_is_visible_to_the_swap_gate(monkeypatch, with_server_tools): + # ``stream`` defaults to false on /v1/messages, so the non-streaming plain and server-tool branches + # are the common shape and decode throughout. Only their streaming siblings registered. + _route_gate() + import asyncio + + from models.inference import AnthropicMessagesRequest + + seen = {} + + def _sample(): + # Sampled mid-generation: exactly the window an /unload lands in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + + def _gen_plain(*, cancel_event = None, **kwargs): + _sample() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield "ok" + + def _gen_tools(*, cancel_event = None, **kwargs): + _sample() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield {"type": "content", "text": "ok"} + + inf_mod = _anthropic_route_stubs( + monkeypatch, + generate_chat_completion = _gen_plain, + generate_chat_completion_with_tools = _gen_tools, + ) + + fields = {"max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} + if with_server_tools: + fields["enable_tools"] = True + fields["tools"] = [{"type": "web_search_20250305", "name": "web_search"}] + payload = AnthropicMessagesRequest(**fields) + + response = asyncio.run( + inf_mod.anthropic_messages(payload, request = _MessagesRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # The event registered is the one the decode watches, so a forced swap lands. + assert seen["reached_the_decode"] + # And it unregisters, or one message would 409 every later reload. + assert active_generations.count() == 0 + + +def test_anthropic_passthrough_non_stream_is_visible_to_the_swap_gate(monkeypatch): + # The client-tool pass-through holds llama-server for one non-streaming POST. Its streaming sibling + # registers inside the body generator; this branch had none, so /unload tore the server down. + _route_gate() + import asyncio + + import httpx + + from models.inference import AnthropicMessagesRequest + + seen = {} + + def handler(request): + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + return httpx.Response( + 200, + json = { + "choices": [ + {"message": {"role": "assistant", "content": "33"}, "finish_reason": "stop"} + ] + }, + ) + + inf_mod = _anthropic_route_stubs(monkeypatch) + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + # The pass-through takes a per-request client, so a Stop or forced swap can close it mid-POST. + monkeypatch.setattr( + inf_mod, + "_cancelable_nonstreaming_client", + lambda: real_async_client(transport = transport), + ) + + # enable_tools False keeps the server-tool loop out, so the client tool takes the pass-through. + payload = AnthropicMessagesRequest( + max_tokens = 16, + messages = [{"role": "user", "content": "hi"}], + enable_tools = False, + tools = [{"name": "lookup", "input_schema": {"type": "object", "properties": {}}}], + ) + + response = asyncio.run( + inf_mod.anthropic_messages(payload, request = _MessagesRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # And it unregisters, or one message would 409 every later reload. + assert active_generations.count() == 0 + + +def test_anthropic_passthrough_non_stream_stops_when_the_swap_cancels_it(monkeypatch): + # Registering is half the job: a pooled client cannot be closed, so the run was cancelled while the + # POST carried on. The watcher closes a per-request client; the set event makes that error a cancel. + _route_gate() + import asyncio + + import httpx + + from models.inference import AnthropicMessagesRequest + + seen = {} + + def handler(request): + # Stand in for a forced swap mid-decode: cancel, then fail the transport as closing would. + seen["cancelled"] = active_generations.cancel_all() + raise httpx.ConnectError("client closed") + + inf_mod = _anthropic_route_stubs(monkeypatch) + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod, + "_cancelable_nonstreaming_client", + lambda: real_async_client(transport = transport), + ) + + payload = AnthropicMessagesRequest( + max_tokens = 16, + messages = [{"role": "user", "content": "hi"}], + enable_tools = False, + tools = [{"name": "lookup", "input_schema": {"type": "object", "properties": {}}}], + ) + + with pytest.raises(asyncio.CancelledError): + asyncio.run( + inf_mod.anthropic_messages( + payload, request = _MessagesRequest(), current_subject = "tester" + ) + ) + + assert seen["cancelled"] == 1 + # Cancelled or not, the entry must go, or one message 409s every later reload. + assert active_generations.count() == 0 + + +def test_audio_generation_unregisters_when_it_fails(monkeypatch): + # A raising backend must not strand an entry: that would 409 every later load. + _route_gate() + import asyncio + from types import SimpleNamespace + + from fastapi import HTTPException + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + class _BrokenTtsBackend: + active_model_name = "org/TTS" + models = {"org/TTS": {"is_audio": True}} + + def generate_audio_response(self, **kwargs): + raise RuntimeError("codec exploded") + + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace(is_loaded = False, _is_audio = False), + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: _BrokenTtsBackend()) + + async def _no_auto_switch(*a, **k): + return None + + monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch) + + payload = ChatCompletionRequest( + model = "org/TTS", + messages = [{"role": "user", "content": "hi"}], + ) + with pytest.raises(HTTPException): + asyncio.run(inf_mod.generate_audio(payload, request = None, current_subject = "tester")) + + assert active_generations.count() == 0 + + +# ── sidecar install: carrying a confirmed swap through ───────────────── + + +def _stub_install_route(monkeypatch, *, in_flight_events): + """Point POST /install-latest-transformers at an in-memory sidecar install. + + ``in_flight_events`` stands in for the middleware's in-flight count: a + request is counted until its stream observes the cancel event and unwinds, + which is the coupling the installer's guard actually reads. + """ + from types import SimpleNamespace + + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + import utils.transformers_latest as latest_mod + import utils.transformers_version as version_mod + + calls = {"installed": [], "released": 0} + + monkeypatch.setattr(version_mod, "try_begin_sidecar_swap", lambda: True) + + def _end_sidecar_swap(): + calls["released"] += 1 + + monkeypatch.setattr(version_mod, "end_sidecar_swap", _end_sidecar_swap) + + import core.export as export_mod + import core.training as training_mod + + monkeypatch.setattr( + training_mod, + "get_training_backend", + lambda: SimpleNamespace(is_training_active = lambda: False), + ) + monkeypatch.setattr( + export_mod, + "get_export_backend", + lambda: SimpleNamespace(is_export_active = lambda: False, current_checkpoint = None), + ) + monkeypatch.setattr( + inf_mod, + "get_inference_backend", + lambda: SimpleNamespace(active_model_name = None, load_generation = 0), + ) + + def _fake_in_flight(current_request_counted = True, *, include_pending = True): + return sum(1 for ev in in_flight_events if not ev.is_set()) + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _fake_in_flight) + + def _install(version, before_swap, *args, **kwargs): + calls["installed"].append(version) + return {"success": True, "version": version, "message": "installed"} + + monkeypatch.setattr(latest_mod, "install_latest_transformers", _install) + return inf_mod, calls + + +def test_confirmed_install_stops_the_chats_it_was_given_permission_to_stop(monkeypatch): + # The install sits between the swap's "stop N chats" prompt and the /load carrying the + # confirmation, and refuses while those chats run, so a confirmed install cancels them itself. + _route_gate() + import asyncio + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev]) + + with active_generations.ActiveGeneration(ev, thread_id = "t1", model = "org/M-GGUF"): + response = asyncio.run( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True), + "tester", + ) + ) + assert ev.is_set() + + assert response.success is True + assert calls["installed"] == ["5.0.0"] + + +def test_unconfirmed_install_still_refuses_while_chats_stream(monkeypatch): + # Unchanged for every caller that never confirmed (second tab, desktop, curl): no flag, no cancel. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev]) + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0"), + "tester", + ) + ) + assert not ev.is_set() + assert active_generations.count() == 1 + + assert exc.value.status_code == 409 + assert calls["installed"] == [] + + +def test_a_confirmed_install_that_cannot_drain_refuses_instead_of_swapping(monkeypatch): + # A cancelled request that never observes its event keeps the in-flight count up, so the drain is + # bounded and cannot wedge the process holding the gate; the recheck behind it still refuses. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + stuck = threading.Event() + stuck.set() # already "cancelled", yet still counted: it never unwinds + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev, stuck]) + monkeypatch.setattr(inf_mod, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.05) + + def _never_unwinds(current_request_counted = True, *, include_pending = True): + return 1 + + import core.inference.llama_keepwarm as keepwarm + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _never_unwinds) + + async def _install(): + # Deadline here too: a regression that drops the drain's bound must fail, not hang the suite. + return await asyncio.wait_for( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True), + "tester", + ), + timeout = 5, + ) + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run(_install()) + + assert exc.value.status_code == 409 + assert calls["installed"] == [] + + +def test_confirmed_install_does_not_spend_its_cancel_on_an_install_that_will_refuse(monkeypatch): + # An unrelated counted request the cancel cannot stop must be waited out BEFORE the cancel: the + # recheck refuses while it is there, so cancelling first stopped chats for a doomed install. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev]) + + import core.inference.llama_keepwarm as keepwarm + + def _never_drains(current_request_counted = True, *, include_pending = True): + # Discounting the registered chat still leaves the counted-only stranger: the drain must not clear. + return 2 + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _never_drains) + monkeypatch.setattr(inf_mod, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.05) + + async def _install(): + return await asyncio.wait_for( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True), + "tester", + ), + timeout = 5, + ) + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run(_install()) + # The refusal is the same as before; what changed is that the chat lives. + assert not ev.is_set() + assert active_generations.count() == 1 + + assert exc.value.status_code == 409 + assert calls["installed"] == [] + + +# ── draining before teardown ────────────────────────────────────────── + + +def _drain_with_counts(monkeypatch, counts, **kwargs): + """Run _wait_for_model_switch_idle against a scripted in-flight count. + + ``counts`` is consumed one entry per poll; the last value repeats, so a + trailing non-zero stands for a request that never unwinds. + """ + _route_gate() + import asyncio + + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + remaining = list(counts) + polls = {"n": 0} + + def _count(current_request_counted = True, *, include_pending = True): + polls["n"] += 1 + return remaining.pop(0) if len(remaining) > 1 else remaining[0] + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0) + + async def _run(): + # Hard test-side deadline: a drain that regresses to waiting forever must fail red, not hang. + await asyncio.wait_for( + inf_mod._wait_for_model_switch_idle(current_request_counted = False, **kwargs), + timeout = 5, + ) + + asyncio.run(_run()) + return polls["n"] + + +def test_forced_swap_does_not_wait_out_the_generations_it_is_about_to_cancel(monkeypatch): + # cancel_pending discounts the registered generations, since the caller cancels them right after. + # Drop the discount and the drain waits on a count only that pending cancel can lower: forever. + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + polls = _drain_with_counts(monkeypatch, [1], cancel_pending = True) + assert polls == 1 + + +def test_the_same_drain_without_the_discount_would_keep_waiting(monkeypatch): + # The other half: that count really does block, so the previous test passes by the discount. + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + polls = _drain_with_counts(monkeypatch, [1], timeout_s = 0.05) + assert polls > 1 + + +def test_post_cancel_drain_gives_up_on_a_request_that_never_unwinds(monkeypatch): + # TTS on the subprocess backend observes no cancel event, so a forced swap can cancel it and still + # see it counted forever. The post-cancel drains hold the gate, so they must expire and proceed. + polls = _drain_with_counts(monkeypatch, [1], timeout_s = 0.05) + assert polls > 1 + + +def test_drain_returns_as_soon_as_the_cancelled_requests_unwind(monkeypatch): + # The bound is a backstop: once the count drops the drain returns without sitting out the timeout. + polls = _drain_with_counts(monkeypatch, [2, 1, 0], timeout_s = 30) + assert polls == 3 + + +# ── queued chats must not cancel the running one ────────────────────── + + +def _orchestrator_for_ownership(): + """A real InferenceOrchestrator with just enough stubbed to drive the lock.""" + _route_gate() + orch_mod = pytest.importorskip( + "core.inference.orchestrator", reason = "inference stack not installed" + ) + orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator) + orch._gen_lock = threading.Lock() + orch._active_cancel_events = [] + orch._executing_cancel_events = [] + orch._active_cancel_lock = threading.Lock() + orch._cancel_event = threading.Event() + orch._ensure_subprocess_alive = lambda: False # stop before _send_cmd + return orch + + +def test_a_queued_chat_cannot_reset_the_chat_that_is_generating(): + # Safetensors generation serialises on _gen_lock and the worker has ONE cancel event: stopping + # queued chat B reset that shared event and killed running chat A. Scope the reset to the holder. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + + orch._claim_worker(a_event) # A holds the lock ... + orch._mark_worker_started(a_event) # ... and the worker is answering it + orch.reset_generation_state(b_event) # B is queued and gets stopped + assert not orch._cancel_event.is_set() + + orch.reset_generation_state(a_event) # A's own Stop still works + assert orch._cancel_event.is_set() + + +def test_a_global_reset_still_cancels_whatever_is_running(): + # Unload and switch pass nothing: they mean stop everything, else a generation survives teardown. + orch = _orchestrator_for_ownership() + _running = threading.Event() + orch._claim_worker(_running) + orch._mark_worker_started(_running) + orch.reset_generation_state() + assert orch._cancel_event.is_set() + + +def test_a_reset_with_no_generation_running_is_not_dropped(): + # Nothing holds the lock, so no chat to protect: a reset before any generation must still run. + orch = _orchestrator_for_ownership() + orch.reset_generation_state(threading.Event()) + assert orch._cancel_event.is_set() + + +def test_unload_waits_for_a_request_that_is_admitted_but_not_yet_registered(monkeypatch): + # The window between the keep-warm middleware and _TrackedCancel: counted in-flight, absent from + # the registry. Cancelling on the registry alone tore the backend down under an admitted request. + _route_gate() + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + # Counted for two polls, then the request registers/finishes and clears. + remaining = [1, 1, 0] + seen = {} + + def _count(current_request_counted = True, *, include_pending = True): + return remaining.pop(0) if len(remaining) > 1 else remaining[0] + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0) + + torn_down: list[str] = [] + + def _record_teardown(): + seen["counted_at_teardown"] = remaining[0] + torn_down.append("gguf") + + # Registry deliberately empty: this is the unregistered case. + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + unload_model = _record_teardown, + ) + + assert active_generations.count() == 0 + assert torn_down == ["gguf"] + assert seen["counted_at_teardown"] == 0 + assert response.status == "unloaded" + + +def test_a_dispatched_chat_cannot_reset_its_concurrently_dispatched_sibling(): + # Compare-mode / dispatched runs bypass _gen_lock and run concurrently, so with several claimed + # at once a Stop on one must still leave the others alone. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + c_event = threading.Event() + + orch._claim_worker(a_event) + orch._mark_worker_started(a_event) + orch._claim_worker(b_event) + orch._mark_worker_started(b_event) + + orch.reset_generation_state(c_event) # a third, unrelated request + assert not orch._cancel_event.is_set() + + orch.reset_generation_state(b_event) # one of the running pair + assert orch._cancel_event.is_set() + + +def test_releasing_one_generation_leaves_the_other_claimed(): + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + orch._claim_worker(a_event) + orch._mark_worker_started(a_event) + orch._claim_worker(b_event) + orch._mark_worker_started(b_event) + orch._release_worker(a_event) + + orch.reset_generation_state(a_event) # now a stranger + assert not orch._cancel_event.is_set() + + orch._release_worker(b_event) + orch.reset_generation_state(a_event) # nothing running: no one to protect + assert orch._cancel_event.is_set() + + +def test_a_dispatched_request_queued_behind_another_is_not_an_owner(): + # The subprocess runs generations one at a time, so admission is not execution: B can be claimed + # while the worker answers A. Counting B as an owner let its Stop signal the shared event and end A. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + + orch._claim_worker(a_event) + orch._mark_worker_started(a_event) # the worker answered A + orch._claim_worker(b_event) # B is only queued behind it + + orch.reset_generation_state(b_event) + assert not orch._cancel_event.is_set(), "a queued request must not reset A" + + orch._mark_worker_started(b_event) # the worker moves on to B + orch.reset_generation_state(b_event) + assert orch._cancel_event.is_set() + + +def test_a_queued_request_cannot_reset_during_the_other_ones_prefill(): + # Between _send_cmd and the first response A is claimed but not executing; treating that as + # "nobody to protect" let a queued request's Stop kill A mid-prefill. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + + orch._claim_worker(a_event) # A sent its command and is in prefill + orch._claim_worker(b_event) # B is queued behind it + + orch.reset_generation_state(b_event) + assert not orch._cancel_event.is_set(), "B must not reset A during prefill" + + # A's own Stop still works before any token has arrived. + orch.reset_generation_state(a_event) + assert orch._cancel_event.is_set() + + +def test_the_oldest_claim_is_the_one_the_worker_is_prefilling(): + # The command queue is FIFO, so with nothing answering the oldest claim is the executor. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + orch._claim_worker(a_event) + orch._claim_worker(b_event) + orch._release_worker(a_event) + + orch.reset_generation_state(b_event) + assert orch._cancel_event.is_set(), "B is now the oldest claim" + + +def test_claim_order_matches_send_order_under_concurrent_dispatch(): + # _owns_worker reads claim order to decide who is prefilling, so a claim not atomic with the + # enqueue can put A first in the list while B is first in the subprocess queue: stopping A kills B. + _route_gate() + orch_mod = pytest.importorskip( + "core.inference.orchestrator", reason = "inference stack not installed" + ) + orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator) + orch._active_cancel_events = [] + orch._executing_cancel_events = [] + orch._active_cancel_lock = threading.Lock() + orch._send_order_lock = threading.Lock() + + sent: list = [] + barrier = threading.Barrier(4) + + def worker(ev): + barrier.wait(timeout = 10) + with orch._send_order_lock: + orch._claim_worker(ev) + # Stand in for _send_cmd: the enqueue must not be separable from the claim. + sent.append(ev) + + events = [threading.Event() for _ in range(4)] + threads = [threading.Thread(target = worker, args = (e,)) for e in events] + for t in threads: + t.start() + for t in threads: + t.join(timeout = 30) + + assert orch._active_cancel_events == sent, "claim order must equal send order" diff --git a/studio/backend/tests/test_anthropic_admission.py b/studio/backend/tests/test_anthropic_admission.py index de01accd08..d4fcf85a45 100644 --- a/studio/backend/tests/test_anthropic_admission.py +++ b/studio/backend/tests/test_anthropic_admission.py @@ -641,12 +641,13 @@ def test_every_dispatch_site_goes_through_admission(): for node in ast.walk(tree) if isinstance(node, ast.AsyncFunctionDef) and node.name == "anthropic_messages" ) - # The wrappers themselves call _monitored_anthropic; only the dispatch sites count. + # The wrappers themselves call _monitored_anthropic (the non-streaming one + # through the swap-gate tracker); only the dispatch sites count. nested = { node for node in ast.walk(handler) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - and node.name.startswith("_admitted_anthropic") + and node.name.startswith(("_admitted_anthropic", "_tracked_anthropic")) } inner = {id(n) for wrapper in nested for n in ast.walk(wrapper)} @@ -763,12 +764,13 @@ def _passthrough_payload(**fields): return _payload(tools = _CLIENT_TOOLS, enable_tools = False, **fields) -def test_response_pre_start_cleanup_exits_the_passthrough_tracker(monkeypatch): - """A disconnect before the body starts must still exit the cancel tracker. +def test_response_pre_start_cleanup_leaves_no_passthrough_tracker(monkeypatch): + """A disconnect before the body starts must leave no tracker and no slot. - The wrapper replaces the response's own pre-start hook, so it has to chain to - it. Asserting through _CANCEL_REGISTRY rather than the wiring, because the - hook can be present and still be a no-op. + The passthrough registers from inside its body rather than eagerly, so a + generator that never runs registers nothing; the hook still has to hand the + admission slot back. Asserting through _CANCEL_REGISTRY and the pool rather + than the wiring, because the hook can be present and still be a no-op. """ backend = _install_backend(monkeypatch, slots = 1) backend.supports_tool_passthrough = True @@ -778,7 +780,7 @@ def test_response_pre_start_cleanup_exits_the_passthrough_tracker(monkeypatch): response = await anthropic_messages( _passthrough_payload(stream = True), request = _Request(), current_subject = "t" ) - assert inf_mod._CANCEL_REGISTRY, "passthrough should have registered a tracker" + assert inf_mod._CANCEL_REGISTRY == {}, "nothing runs the body's exit for it yet" cleanup = getattr(response, "_unstarted_cleanup", None) assert cleanup is not None diff --git a/studio/backend/tests/test_anthropic_passthrough_respawn.py b/studio/backend/tests/test_anthropic_passthrough_respawn.py index a9f31208ed..daa30e39c2 100644 --- a/studio/backend/tests/test_anthropic_passthrough_respawn.py +++ b/studio/backend/tests/test_anthropic_passthrough_respawn.py @@ -74,6 +74,10 @@ class _Request: class _FakeNonStreamingClient: def __init__(self): self.urls = [] + self.closed = False + + async def aclose(self): + self.closed = True async def post(self, url, **_kwargs): self.urls.append(url) @@ -189,7 +193,7 @@ def test_retry_url_tolerates_a_backend_without_respawn_hooks(): def test_non_streaming_retries_against_the_new_port(monkeypatch): client = _FakeNonStreamingClient() - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) backend = _Backend() response = asyncio.run(_run_non_streaming(backend)) @@ -201,7 +205,7 @@ def test_non_streaming_retries_against_the_new_port(monkeypatch): def test_non_streaming_raises_when_the_server_stays_dead(monkeypatch): client = _FakeNonStreamingClient() - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) backend = _Backend(respawn_ok = False) with pytest.raises(httpx.ConnectError): @@ -212,7 +216,7 @@ def test_non_streaming_raises_when_the_server_stays_dead(monkeypatch): def test_non_streaming_does_not_retry_an_mtp_crash(monkeypatch): client = _FakeNonStreamingClient() - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) backend = _Backend(mtp_handled = True) with pytest.raises(httpx.ConnectError): diff --git a/studio/backend/tests/test_inference_dispatcher_resilience.py b/studio/backend/tests/test_inference_dispatcher_resilience.py index 6184496d78..ea903a6ce0 100644 --- a/studio/backend/tests/test_inference_dispatcher_resilience.py +++ b/studio/backend/tests/test_inference_dispatcher_resilience.py @@ -39,6 +39,7 @@ def _dispatcher(): o._dispatcher_stop = threading.Event() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} return o @@ -118,3 +119,68 @@ def test_route_llama_streaming_async_clients_disable_proxy_env(): kw.arg == "trust_env" and isinstance(kw.value, ast.Constant) and kw.value.value is False for kw in call.keywords ), f"httpx.AsyncClient at line {call.lineno} must set trust_env=False" + + +def _direct_reader_host(): + """Orchestrator with only what _direct_reader and the ownership helpers touch.""" + o = InferenceOrchestrator.__new__(InferenceOrchestrator) + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + o._request_cancel_events = {} + o._active_cancel_lock = threading.Lock() + o._active_cancel_events = [] + o._executing_cancel_events = [] + o._dispatcher_thread = None + return o + + +def test_rerouting_a_foreign_response_moves_worker_ownership(): + # A _gen_lock reader already blocked on resp_queue can beat the compare dispatcher to + # that request's first response. The compare consumer passes mark_started=False, so if + # this path does not promote it nothing does: the direct request stays recorded as the + # executor, so the compare chat's Stop is ignored and a late reset from the direct one + # cancels the compare generation instead. + o = _direct_reader_host() + mine, theirs = threading.Event(), threading.Event() + o._request_cancel_events = {"mine": mine, "theirs": theirs} + o._claim_worker(mine) + o._mark_worker_started(mine) + o._claim_worker(theirs) + compare_mailbox = queue.Queue() + o._mailboxes["theirs"] = compare_mailbox + + read_one, _drain, release = _direct_reader_calls(o, "mine") + o._scripted = [{"request_id": "theirs", "type": "token", "text": "hi"}] + + assert read_one(timeout = 0.1) is None, "a foreign response is routed, not returned" + assert compare_mailbox.get_nowait()["text"] == "hi" + assert o._owns_worker(theirs), "the compare request is the one the worker answered" + assert not o._owns_worker(mine), "so a late reset from the direct request must not fire" + release() + + +def test_rerouting_a_foreign_gen_done_retires_that_request(): + # The other half of the dispatcher's move: once its last response is routed, the + # request no longer owns the worker, or a Stop for it would end whatever starts next. + o = _direct_reader_host() + mine, theirs = threading.Event(), threading.Event() + o._request_cancel_events = {"mine": mine, "theirs": theirs} + o._claim_worker(theirs) + o._mark_worker_started(theirs) + o._claim_worker(mine) + o._mailboxes["theirs"] = queue.Queue() + + read_one, _drain, release = _direct_reader_calls(o, "mine") + o._scripted = [{"request_id": "theirs", "type": "gen_done"}] + + assert read_one(timeout = 0.1) is None + assert not o._owns_worker(theirs), "retired once its last response was routed" + assert o._owns_worker(mine), "the next claim takes over" + release() + + +def _direct_reader_calls(o, request_id): + """_direct_reader wired to a scripted _read_resp (o._scripted, popped in order).""" + o._read_resp = lambda timeout = 1.0: o._scripted.pop(0) if o._scripted else None + return o._direct_reader(request_id) diff --git a/studio/backend/tests/test_llama_admission.py b/studio/backend/tests/test_llama_admission.py index f69eb7c5c9..9ff19ec27d 100644 --- a/studio/backend/tests/test_llama_admission.py +++ b/studio/backend/tests/test_llama_admission.py @@ -847,3 +847,222 @@ def test_dead_waiters_stop_counting_against_the_queue_limit(): assert queue.is_idle() asyncio.run(_run()) + + +def test_parking_frees_the_slot_for_a_waiter(): + """A holder waiting on a tool approval must not hold a decode slot. + + It is not generating, and with several prompts unanswered every slot would + be held by a run parked on a human while llama-server sits idle. + """ + + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + first = queue.reserve(capacity = 1, config = config) + second = queue.reserve(capacity = 1, config = config) + first_lease = first.lease_nowait() + assert first_lease is not None + assert second.lease_nowait() is None + + first_lease.park() + assert first_lease.slot is None, "the slot went back to the pool" + second_lease = await second.wait(0.1) + assert second_lease is not None, "parking did not free the slot" + + # The parked holder keeps its lease, so releasing it is still correct. + first_lease.unpark() + first_lease.release() + second_lease.release() + assert queue.snapshot().active == 0 + + asyncio.run(_run()) + + +def test_unpark_without_park_is_a_no_op(): + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + first = queue.reserve(capacity = 1, config = config) + first_lease = first.lease_nowait() + assert first_lease is not None + first_lease.unpark() + first_lease.unpark() + + second = queue.reserve(capacity = 1, config = config) + assert second.lease_nowait() is None, "capacity leaked past the limit" + + asyncio.run(_run()) + + +def test_releasing_a_parked_lease_leaves_the_queue_evictable(): + # is_idle() drives registry eviction, and a parked holder owns no slot, so + # nothing but the parked count keeps its queue alive. A stuck count would + # pin every dead queue for the life of the process. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + lease = queue.reserve(capacity = 1, config = config).lease_nowait() + lease.park() + assert not queue.is_idle(), "a parked holder is coming back to this queue" + lease.release() + assert queue.is_idle() + + asyncio.run(_run()) + + +def test_unpark_waits_instead_of_putting_two_holders_on_one_slot(): + # park() hands the freed slot to a waiter, so by the time the user answers an approval + # prompt someone else may be decoding in it. Resuming regardless left two holders + # against capacity 1, and the resumed tool loop went past the admission limit. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None, "A takes the only slot" + b = queue.reserve(capacity = 1, config = config) + assert b.lease_nowait() is None, "B waits behind A" + + a_lease.park() # A parks on an approval prompt; its slot goes to B + b_lease = await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) + assert b_lease is not None, "B was granted the parked slot" + + # A answers the prompt while B is still decoding: it must WAIT. + resumed = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.05) + assert not resumed.done(), "A must not resume while B holds the slot" + assert queue.snapshot().active <= 1, "never over capacity while waiting" + + b_lease.release() + await asyncio.wait_for(resumed, timeout = 2) + assert a_lease.slot is not None, "A took a real slot back" + assert queue.snapshot().active <= 1, "still within capacity after resuming" + + asyncio.run(scenario()) + + +def test_unpark_gives_up_when_the_caller_is_cancelled(): + # A holder being torn down must not sit in the wait loop. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + b = queue.reserve(capacity = 1, config = config) + a_lease.park() + assert await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) is not None + + ev = threading.Event() + waiting = asyncio.ensure_future(a_lease.unpark_async(cancel_event = ev, poll_s = 0.01)) + await asyncio.sleep(0.03) + assert not waiting.done() + ev.set() + await asyncio.wait_for(waiting, timeout = 2) + assert a_lease.slot is None, "gave up without a slot rather than over-admitting" + + asyncio.run(scenario()) + + +def test_an_approved_chat_is_not_overtaken_by_later_arrivals(): + # A parks on an approval prompt, B takes the slot, C arrives afterwards. release() grants + # under the same lock, so a plain poll in unpark_async never saw a free slot: A waited + # behind every later arrival and starved. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + b = queue.reserve(capacity = 1, config = config) + a_lease.park() # A's slot goes to B + b_lease = await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) + assert b_lease is not None + + # A is approved and starts waiting; C arrives only after that. + resumed = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.03) + c = queue.reserve(capacity = 1, config = config) + assert c.lease_nowait() is None + + b_lease.release() # the slot frees exactly once + await asyncio.wait_for(resumed, timeout = 2) + # A resumed; C is still queued behind it rather than having overtaken it. + assert c.lease_nowait() is None + assert queue.snapshot().active <= 1 + + asyncio.run(scenario()) + + +def test_two_approved_chats_do_not_block_each_other(): + # A bare pending-count made every approved holder count against every other: park A, admit + # and park B, admit C, approve both, and once C released the predicate stayed false forever. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + b = queue.reserve(capacity = 1, config = config) + a_lease.park() # A parks; B is admitted + b_lease = await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) + assert b_lease is not None + + c = queue.reserve(capacity = 1, config = config) + b_lease.park() # B parks too; C is admitted + c_lease = await asyncio.wait_for(c.wait(timeout_s = 1), timeout = 2) + assert c_lease is not None + + # Both approvals come back while C is still decoding. + first = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.02) + second = asyncio.ensure_future(b_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.02) + assert not first.done() and not second.done() + + c_lease.release() + # The earlier approval goes first; the other follows once it releases. + await asyncio.wait_for(first, timeout = 2) + assert not second.done(), "the second approval waits its turn, not forever" + a_lease.release() + await asyncio.wait_for(second, timeout = 2) + assert queue.snapshot().active <= 1 + + asyncio.run(scenario()) + + +def test_an_immediate_arrival_cannot_take_an_approved_chats_slot(): + # The fairness reservation lived only in _grant_waiters_locked. reserve()'s fast path + # ignored it, so a request arriving in the window between the slot freeing and the + # approved chat's next poll took the slot straight off the top. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + a_lease.park() # A is on an approval prompt; its slot is up for grabs + b = queue.reserve(capacity = 1, config = config) + b_lease = b.lease_nowait() + assert b_lease is not None + + resumed = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.03) # A is approved and now holds a ticket + + # No await between these two: C arrives before A's poll can run again. + b_lease.release() + c = queue.reserve(capacity = 1, config = config) + assert c.lease_nowait() is None, "the freed slot is reserved for the approved chat" + + await asyncio.wait_for(resumed, timeout = 2) + assert queue.snapshot().active <= 1 + + asyncio.run(scenario()) diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py index cf41d540f1..7b20063892 100644 --- a/studio/backend/tests/test_llama_cpp_tool_loop.py +++ b/studio/backend/tests/test_llama_cpp_tool_loop.py @@ -2076,6 +2076,50 @@ def test_large_python_tool_call_emits_early_provisional_start(monkeypatch): assert any(e.get("type") == "tool_end" and e.get("tool_name") == "python" for e in events) +def test_gated_python_call_still_streams_its_arguments(monkeypatch): + """A call awaiting approval still streams its code into the card. + + Suppressing it left the chat completely blank for as long as the model took + to write the payload, which for a large file is minutes. Nothing runs before + the decision either way, and the code is what the user is approving. + """ + + big_code = "total = 0\n" + "\n".join(f"total += {i}" for i in range(120)) + assert len(json.dumps({"code": big_code})) > _PROVISIONAL_ARGS_MIN_CHARS + + first_stream = _streamed_structured_tool_call("python", {"code": big_code}, "call_gated") + final_stream = [_sse({"content": "Done."}), _done()] + payloads: list[dict] = [] + backend = _make_backend(monkeypatch, [first_stream, final_stream], payloads) + + monkeypatch.setattr("core.inference.tools.execute_tool", lambda name, arguments, **_k: "OK") + monkeypatch.setattr("core.inference.llama_cpp.wait_tool_decision", lambda *_a, **_k: "allow") + + events = list( + backend.generate_chat_completion_with_tools( + messages = [{"role": "user", "content": "write code"}], + tools = [{"type": "function", "function": {"name": "python"}}], + confirm_tool_calls = True, + permission_mode = "ask", + max_tool_iterations = 1, + ) + ) + + tool_starts = [e for e in events if e.get("type") == "tool_start"] + provisional = [e for e in tool_starts if not e.get("arguments")] + assert len(provisional) == 1, tool_starts + assert provisional[0]["tool_call_id"] == "call_gated" + + args_events = [e for e in events if e.get("type") == "tool_args"] + assert args_events, "gated call streamed no arguments" + assert "total += 119" in "".join(e["text"] for e in args_events) + + # The approval prompt still fires, and it comes after the code is on screen. + gated = [e for e in tool_starts if e.get("awaiting_confirmation")] + assert gated, tool_starts + assert events.index(provisional[0]) < events.index(gated[0]) + + def test_auto_mode_render_html_suppresses_provisional_card_under_confirm(monkeypatch): """render_html is no longer unconditionally safe (a networked canvas asks), so with confirm_tool_calls set under permission_mode="auto" its early provisional diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 065eddfe99..e29fc07a95 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -1606,22 +1606,28 @@ def test_load_route_holds_lifecycle_gate(monkeypatch): def test_model_replacements_recheck_sidecar_swap_before_either_backend_is_unloaded(): - # Both replacement directions drain active inference, then recheck whether a - # sidecar install reserved the lifecycle gate during that wait. Exact-model - # reuse exits earlier, so an already-loaded model never waits on unrelated inference. + # Both replacement directions drain, then recheck whether a sidecar install reserved the + # gate meanwhile. That recheck is the last thing that can reject the load, so the + # destructive cancel must follow it. Exact-model reuse exits earlier and never waits. import inspect src = inspect.getsource(inference_route._load_model_impl) + already_loaded = src.index('status = "already_loaded"') + standard_branch = src.index("# ── Standard path") + gguf_wait = src.index("await _wait_for_model_switch_idle", src.index("if config.is_gguf:")) gguf_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", gguf_wait) + gguf_cancel = src.index("on_reload_confirmed(cancel = True)", gguf_wait) unload_unsloth = src.index("unsloth_backend.unload_model", gguf_wait) - standard_wait = src.index("await _wait_for_model_switch_idle", gguf_wait + 1) - standard_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", standard_wait) - unload_gguf = src.index("llama_backend.unload_model()", standard_wait) - already_loaded = src.index('status = "already_loaded"') - assert already_loaded < gguf_wait < gguf_sidecar_check < unload_unsloth - assert standard_wait < standard_sidecar_check < unload_gguf + standard_wait = src.index("await _wait_for_model_switch_idle", standard_branch) + standard_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", standard_wait) + standard_cancel = src.index("on_reload_confirmed(cancel = True)", standard_wait) + unload_gguf = src.index("llama_backend.unload_model()", standard_wait) + + assert already_loaded < gguf_wait < gguf_sidecar_check < gguf_cancel < unload_unsloth + assert standard_branch < standard_wait < standard_sidecar_check + assert standard_sidecar_check < standard_cancel < unload_gguf def test_switch_waiter_deregisters_before_swap_gate_release(): diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 7758339070..eeb6cee871 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -4615,6 +4615,9 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"prompt": "hi", "stream": False} + async def is_disconnected(self): + return False + class FailingAsyncClient: async def __aenter__(self): return self @@ -4622,14 +4625,18 @@ class TestApiMonitorProviderAndCompletionStreams: async def __aexit__(self, *_args): return False + async def aclose(self): + return None + async def post(self, *_args, **_kwargs): raise httpx.ConnectError("llama down") monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) + # Per-request client so a forced swap can close it mid-call; the pooled one is shared. monkeypatch.setattr( inf_mod, - "nonstreaming_client", + "_cancelable_nonstreaming_client", lambda: FailingAsyncClient(), ) monkeypatch.setattr( @@ -4667,9 +4674,15 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"prompt": "hi", "stream": False} + async def is_disconnected(self): + return False + captured = [] class CapturingClient: + async def aclose(self): + return None + async def post(self, _url, *, json, **_kwargs): captured.append(dict(json)) return httpx.Response( @@ -4687,7 +4700,9 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: CapturingClient()) + monkeypatch.setattr( + inf_mod, "_cancelable_nonstreaming_client", lambda: CapturingClient() + ) monkeypatch.setattr( inf_mod, "get_llama_cpp_backend", @@ -4718,9 +4733,15 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"prompt": "hi", "stream": False, "max_tokens": 0} + async def is_disconnected(self): + return False + captured = [] class CapturingClient: + async def aclose(self): + return None + async def post(self, _url, *, json, **_kwargs): captured.append(dict(json)) return httpx.Response( @@ -4738,7 +4759,9 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: CapturingClient()) + monkeypatch.setattr( + inf_mod, "_cancelable_nonstreaming_client", lambda: CapturingClient() + ) monkeypatch.setattr( inf_mod, "get_llama_cpp_backend", @@ -4776,6 +4799,7 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: UnusedClient()) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: UnusedClient()) monkeypatch.setattr( inf_mod, "get_llama_cpp_backend", @@ -4880,6 +4904,9 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"input": ["alpha", "beta"], "model": "embed"} + async def is_disconnected(self): + return False + class FakeAsyncClient: async def __aenter__(self): return self @@ -4887,6 +4914,9 @@ class TestApiMonitorProviderAndCompletionStreams: async def __aexit__(self, *_args): return False + async def aclose(self): + return None + async def post(self, *_args, **_kwargs): assert monitor.active_count() == 1 return httpx.Response( @@ -4899,9 +4929,10 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) + # Per-request client so a forced swap can close it mid-call; the pooled one is shared. monkeypatch.setattr( inf_mod, - "nonstreaming_client", + "_cancelable_nonstreaming_client", lambda: FakeAsyncClient(), ) monkeypatch.setattr( @@ -6372,7 +6403,7 @@ class TestApiMonitorSafetensorsUsage: } yield "safe reply" - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): pass monitor = ApiMonitor(max_entries = 3) @@ -6443,7 +6474,7 @@ class TestApiMonitorSafetensorsUsage: cancel_event.set() yield {"type": "content", "text": "ignored"} - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): pass monitor = ApiMonitor(max_entries = 3) @@ -6504,7 +6535,7 @@ class TestApiMonitorSafetensorsUsage: def generate_chat_completion_with_tools(self, **_kwargs): yield {"type": "content", "text": "unused"} - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): nonlocal reset_called reset_called = True diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py index 3a36500aee..7963b71e8e 100644 --- a/studio/backend/tests/test_orchestrator_unload_cancel.py +++ b/studio/backend/tests/test_orchestrator_unload_cancel.py @@ -19,6 +19,10 @@ def _bare_orchestrator(): """An orchestrator without the real __init__ subprocess/network.""" o = InferenceOrchestrator.__new__(InferenceOrchestrator) o._gen_lock = threading.Lock() + o._send_order_lock = threading.Lock() + o._active_cancel_lock = threading.Lock() + o._active_cancel_events = [] + o._executing_cancel_events = [] o._cancel_event = threading.Event() # stands in for the mp.Event o._drain_event = threading.Event() # stands in for the unload-drain mp.Event o._proc = object() # truthy so _ensure_subprocess_alive reports alive @@ -775,6 +779,7 @@ def test_dispatched_bails_when_unload_flips_before_mailbox_registration(monkeypa o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) monkeypatch.setattr(o, "_start_dispatcher", lambda: None) @@ -817,6 +822,7 @@ def test_dispatched_bails_when_model_swapped_before_mailbox_registration(monkeyp o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -846,6 +852,7 @@ def test_dispatched_bails_when_dispatcher_stopped_before_mailbox_registration(mo o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -872,6 +879,7 @@ def test_dispatched_happy_path_registers_and_sends(monkeypatch): o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1338,6 +1346,7 @@ def test_dispatched_bail_stops_orphan_dispatcher_it_started(monkeypatch): o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = None # none running -> this call starts it monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1382,6 +1391,7 @@ def test_dispatched_bail_keeps_dispatcher_with_other_active_mailbox(monkeypatch) o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = None monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1419,6 +1429,7 @@ def test_dispatched_bail_keeps_preexisting_dispatcher(monkeypatch): o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() # already running monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1545,6 +1556,7 @@ def test_concurrent_start_dispatcher_spawns_exactly_one(): o._resp_queue = _queue.Queue() # real queue so the dispatcher loop blocks and stays alive o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._dispatcher_thread = None o._dispatcher_stop = threading.Event() o._dispatcher_lifecycle_lock = threading.Lock() @@ -1660,6 +1672,7 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher(): o._resp_queue = _queue.Queue() # a spawned dispatcher would block-read here and stay alive o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._dispatcher_stop = threading.Event() o._dispatcher_lifecycle_lock = threading.Lock() o._unload_pending = False @@ -1713,3 +1726,310 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher(): assert o._dispatcher_thread is None, "the stop cleared it and the queued start spawned nothing" live = [t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()] assert live == [], "no fresh dispatcher may be left to consume the unloaded reply" + + +def _dispatch(o, resps): + """Run the dispatcher over a fixed response list and stop it.""" + import queue as _queue + + o._resp_queue = _queue.Queue() + for r in resps: + o._resp_queue.put(r) + o._dispatcher_stop = threading.Event() + t = threading.Thread(target = o._dispatcher_loop, daemon = True) + t.start() + deadline = time.monotonic() + 5.0 + while not o._resp_queue.empty() and time.monotonic() < deadline: + time.sleep(0.01) + o._dispatcher_stop.set() + t.join(timeout = 5.0) + + +def test_worker_ownership_follows_the_worker_not_the_consumer(): + # The subprocess runs one generation at a time and can start B while A's consumer has yet to + # drain its mailbox. A must stop owning the worker the moment its gen_done is routed, else + # a late Stop for A cancels B. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._mailboxes = {"a": _queue.Queue(), "b": _queue.Queue()} + o._request_cancel_events = {"a": a_cancel, "b": b_cancel} + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + + _dispatch(o, [{"type": "token", "request_id": "a", "token": "hi"}]) + assert o._owns_worker(a_cancel), "the request the worker is answering owns it" + assert not o._owns_worker(b_cancel), "a queued request does not" + + # A finishes. B has been sent but has not answered yet (it is prefilling), so the gap + # between the two is the window a late Stop for A used to fire into. + _dispatch(o, [{"type": "gen_done", "request_id": "a"}]) + assert not o._owns_worker(a_cancel), "a finished request stops owning the worker" + assert o._owns_worker(b_cancel), "the next queued request is the one prefilling" + + # Worker moves on to B, still before A's consumer reads anything. + _dispatch(o, [{"type": "token", "request_id": "b", "token": "yo"}]) + assert not o._owns_worker(a_cancel), "a finished request must not cancel its successor" + assert o._owns_worker(b_cancel), "the worker moved on to B, so B owns it" + + # A's own stream unwinding afterwards must not disturb B. + o._release_worker(a_cancel) + assert o._owns_worker(b_cancel) + + +def test_status_responses_do_not_transfer_worker_ownership(): + # Status lines are not an answer to any request; the dispatcher drops them before routing. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._mailboxes = {"a": _queue.Queue(), "b": _queue.Queue()} + o._request_cancel_events = {"a": a_cancel, "b": b_cancel} + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + + _dispatch(o, [{"type": "status", "request_id": "b", "message": "loading"}]) + # Nothing has answered, so the oldest claim is still the one prefilling. + assert o._owns_worker(a_cancel) + assert not o._owns_worker(b_cancel) + + +def test_only_the_latest_responder_executes(): + # The subprocess runs one generation at a time, so answering B means it has left A. + # _generate_inner promotes from its own consumer and can share the worker with a + # dispatched request, so the two must not both count as executing. + o = _bare_orchestrator() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + + o._mark_worker_started(a_cancel) + assert o._owns_worker(a_cancel) + o._mark_worker_started(b_cancel) + assert o._owns_worker(b_cancel), "the latest responder is the one executing" + assert not o._owns_worker(a_cancel), "and it is the only one" + # Idempotent: more of B's own tokens must not disturb it. + o._mark_worker_started(b_cancel) + assert o._owns_worker(b_cancel) + + +def test_a_stale_mailbox_read_does_not_cancel_the_running_generation(): + # A dispatched consumer can still be draining tokens after the dispatcher retired its request + # and started the next one. Stopping it then must tear down only its own stream: signalling + # the shared worker event would end its successor. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._mailboxes = {"a": _queue.Queue(), "b": _queue.Queue()} + o._request_cancel_events = {"a": a_cancel, "b": b_cancel} + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + # Worker finished A and moved on to B. + _dispatch( + o, + [ + {"type": "gen_done", "request_id": "a"}, + {"type": "token", "request_id": "b", "token": "yo"}, + ], + ) + assert o._owns_worker(b_cancel) and not o._owns_worker(a_cancel) + + # A's consumer now reads a token buffered before that, with A stopped. + a_cancel.set() + stale = [{"type": "token", "request_id": "a", "text": "late"}] + drained = [] + list( + o._consume_token_stream( + lambda timeout: stale.pop(0) if stale else None, + lambda: drained.append(True), + crash_context = "generation", + cancel_event = a_cancel, + mark_started = False, + ) + ) + assert drained, "the stopped stream still tears itself down" + assert not o._cancel_event.is_set(), "a retired request must not signal the shared worker event" + + # The generation that does own the worker still can. + b_cancel.set() + stale_b = [{"type": "token", "request_id": "b", "text": "live"}] + list( + o._consume_token_stream( + lambda timeout: stale_b.pop(0) if stale_b else None, + lambda: None, + crash_context = "generation", + cancel_event = b_cancel, + mark_started = False, + ) + ) + assert o._cancel_event.is_set(), "the running generation's own Stop must reach the worker" + + +def test_a_dispatcher_started_mid_stream_still_reaches_the_direct_reader(): + # A compare request can start the dispatcher while an ordinary chat is streaming. The + # dispatcher then owns resp_queue, and without a mailbox for the direct reader it dropped + # that chat's tokens and its gen_done as unaddressed, hanging it. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + o._request_cancel_events = {} + + read_one, _drain, release = o._direct_reader("direct-1") + try: + _dispatch( + o, + [ + {"type": "token", "request_id": "direct-1", "text": "hi"}, + {"type": "gen_done", "request_id": "direct-1"}, + ], + ) + assert read_one(timeout = 0.1) == { + "type": "token", + "request_id": "direct-1", + "text": "hi", + }, "the dispatcher must route to the direct reader, not drop" + assert read_one(timeout = 0.1)["type"] == "gen_done" + finally: + release() + assert o._direct_mailboxes == {}, "the mailbox is dropped when the stream ends" + + +def test_the_direct_reader_hands_back_a_compare_response_it_took(): + # The mirror race: this reader is already blocked on resp_queue when a compare request's + # dispatcher starts, so it can take that request's response first. Consuming it would + # corrupt this chat and hang the compare pane. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + compare_box: _queue.Queue = _queue.Queue() + o._mailboxes = {"compare-1": compare_box} + o._direct_mailboxes = {} + o._request_cancel_events = {} + o._resp_queue = _queue.Queue() + o._dispatcher_thread = None # no dispatcher yet: this reader owns the queue + + read_one, _drain, release = o._direct_reader("direct-1") + try: + o._resp_queue.put({"type": "token", "request_id": "compare-1", "text": "theirs"}) + o._resp_queue.put({"type": "token", "request_id": "direct-1", "text": "mine"}) + assert read_one(timeout = 0.1) is None, "a foreign response is not ours to yield" + assert compare_box.get_nowait()["text"] == "theirs", "it goes to its own mailbox" + assert read_one(timeout = 0.1)["text"] == "mine" + finally: + release() + + +def test_a_direct_mailbox_is_not_mistaken_for_compare_activity(): + # _mailboxes means "compare requests are in flight" to the unload and distributed paths, + # so an ordinary chat's mailbox must live somewhere else. + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + _read_one, _drain, release = o._direct_reader("direct-1") + try: + assert o._mailboxes == {} + assert "direct-1" in o._direct_mailboxes + finally: + release() + + +def test_replacing_the_subprocess_clears_worker_scoped_state(): + # Ownership is keyed only by cancel-event identity, so a consumer still blocked on its + # mailbox when the worker was replaced stayed recorded as the executor. A generation on + # the fresh worker then failed _owns_worker and could not be stopped. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + dead = threading.Event() + o._mailboxes = {"compare-1": _queue.Queue()} + o._direct_mailboxes = {"direct-1": _queue.Queue()} + o._request_cancel_events = {"compare-1": dead} + o._claim_worker(dead) + o._mark_worker_started(dead) + assert o._owns_worker(dead) + + o._reset_worker_scoped_state() + + assert o._mailboxes == {} and o._direct_mailboxes == {} + assert o._request_cancel_events == {} + assert o._active_cancel_events == [] and o._executing_cancel_events == [] + # A generation on the fresh worker owns it rather than being refused by a ghost. + fresh = threading.Event() + o._claim_worker(fresh) + assert o._owns_worker(fresh), "the dead worker's request must not outrank a live one" + + +def test_audio_input_claims_the_worker_before_sending(): + # Unclaimed, a compare request queued behind an audio-input generation looked like the + # oldest owner, so stopping that queued request signalled the worker and killed this. + import ast + import pathlib + + src = pathlib.Path(orch_mod.__file__).read_text(encoding = "utf-8") + tree = ast.parse(src) + fn = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) and n.name == "_generate_audio_input_inner" + ) + body = ast.get_source_segment(src, fn) or "" + claim = body.find("self._claim_worker(cancel_event)") + send = body.find("self._send_cmd(cmd)") + assert claim != -1, "_generate_audio_input_inner must claim the worker" + assert send != -1 + assert claim < send, "the claim has to happen before the command is enqueued" + assert "with self._send_order_lock:" in body, "claim and send must be one critical section" + assert "self._release_worker(cancel_event)" in body + + +def test_generation_stopped_while_queued_is_never_sent(monkeypatch): + # Two chats on the serialized backend: the second blocks on _gen_lock, and Stop sets its + # event while it waits. Sending anyway occupied the worker with a run the user ended -- + # the cancel is only checked on a token, so a long prefill (or a generation that reaches + # gen_done without one) still held up its siblings. + o = _bare_orchestrator() + monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) + monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda *a, **k: None) + monkeypatch.setattr( + o, "_send_cmd", lambda cmd: pytest.fail("must not send a generation already stopped") + ) + stopped = threading.Event() + stopped.set() + + out = list( + o._generate_inner(messages = [{"role": "user", "content": "hi"}], cancel_event = stopped) + ) + + assert out == [], "a stopped request yields nothing rather than an error banner" + assert o._active_cancel_events == [], "it must not claim the worker either" + assert o._gen_lock.acquire(blocking = False) + o._gen_lock.release() + + +def test_audio_input_stopped_while_queued_is_never_sent(monkeypatch): + # Same lock, same hole. + o = _bare_orchestrator() + monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) + monkeypatch.setattr( + o, "_send_cmd", lambda cmd: pytest.fail("must not send a generation already stopped") + ) + stopped = threading.Event() + stopped.set() + + out = list(o._generate_audio_input_inner(audio_array = [0.0, 0.1], cancel_event = stopped)) + + assert out == [] + assert o._active_cancel_events == [] + assert o._gen_lock.acquire(blocking = False) + o._gen_lock.release() diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py index da261e8d0d..5a01839914 100644 --- a/studio/backend/tests/test_passthrough_healing.py +++ b/studio/backend/tests/test_passthrough_healing.py @@ -504,11 +504,12 @@ def _upstream_message( class ScriptedClient: - """Fake nonstreaming_client() returning scripted JSON bodies, counting POSTs.""" + """Fake upstream client returning scripted JSON bodies, counting POSTs.""" def __init__(self, bodies): self.bodies = list(bodies) self.posts = [] + self.closed = False async def post( self, @@ -520,6 +521,10 @@ class ScriptedClient: self.posts.append(json) return httpx.Response(200, json = self.bodies[min(len(self.posts) - 1, len(self.bodies) - 1)]) + async def aclose(self): + # The Anthropic pass-through owns its client and closes it in a finally. + self.closed = True + async def _drive_non_streaming(monkeypatch, payload, bodies): import routes.inference as inf_mod @@ -867,7 +872,7 @@ class TestNudgeRetryAnthropic: from routes.inference import _anthropic_passthrough_non_streaming client = ScriptedClient(bodies) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) response = await _anthropic_passthrough_non_streaming( _llama_backend(), [{"role": "user", "content": "hi"}], @@ -925,7 +930,7 @@ class TestAnthropicPassthroughHealingText: from routes.inference import _anthropic_passthrough_non_streaming client = ScriptedClient([upstream]) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) response = await _anthropic_passthrough_non_streaming( _llama_backend(), [{"role": "user", "content": "hi"}], @@ -1171,7 +1176,7 @@ class TestAnthropicNonStreamingRoute: from routes.inference import _anthropic_passthrough_non_streaming client = ScriptedClient(bodies) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) response = await _anthropic_passthrough_non_streaming( _llama_backend(), [{"role": "user", "content": "hi"}], diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.py index bb18acf6e5..1043005f64 100644 --- a/studio/backend/tests/test_safetensors_tool_loop.py +++ b/studio/backend/tests/test_safetensors_tool_loop.py @@ -5051,3 +5051,27 @@ class TestFalseAlarmMarkerProse: assert [c[0] for c in exec_fn.calls] == ["web_search", "python"] assistant = next(m for m in convs[1] if m["role"] == "assistant") assert '"python"' not in (assistant.get("content") or "") + + +def test_both_tool_loops_say_they_are_waiting_for_approval(): + """A gated call must not report "Running" in either loop. + + The GGUF loop was fixed first and the safetensors one was missed, so the + badge counted up "Running ..." against a prompt nobody had answered yet. + Asserted on the source so the two paths cannot drift apart again. + """ + import ast + import os + + backend = os.path.join(os.path.dirname(__file__), "..") + for name in ("core/inference/safetensors_agentic.py", "core/inference/llama_cpp.py"): + with open(os.path.join(backend, name), encoding = "utf-8") as f: + tree = ast.parse(f.read()) + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "awaiting_approval_status" + ] + assert calls, f"{name} still announces a gated tool call as running" diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py index f91eec9817..3cc7d0604f 100644 --- a/studio/backend/tests/test_sf_client_tools_passthrough.py +++ b/studio/backend/tests/test_sf_client_tools_passthrough.py @@ -95,7 +95,7 @@ class _ScriptedBackend: for snap in snapshots: yield snap - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): self.reset_count += 1 diff --git a/studio/backend/tests/test_shutdown_preserves_live_worker.py b/studio/backend/tests/test_shutdown_preserves_live_worker.py index faf273411c..15ef93c002 100644 --- a/studio/backend/tests/test_shutdown_preserves_live_worker.py +++ b/studio/backend/tests/test_shutdown_preserves_live_worker.py @@ -9,6 +9,8 @@ holds sidecar transformers modules (breaking the rename on Windows). The methods the handle and return False so callers can refuse the swap. """ +import threading + import pytest from core.export.orchestrator import ExportOrchestrator @@ -52,6 +54,14 @@ def _bare_inference(): o._resp_queue = _Q() o._cancel_event = None o._drain_event = None + # Worker-scoped bookkeeping the teardown clears (see _reset_worker_scoped_state). + o._active_cancel_lock = threading.Lock() + o._active_cancel_events = [] + o._executing_cancel_events = [] + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + o._request_cancel_events = {} return o diff --git a/studio/backend/tests/test_tool_sandbox_per_thread.py b/studio/backend/tests/test_tool_sandbox_per_thread.py new file mode 100644 index 0000000000..13bd95c9ed --- /dev/null +++ b/studio/backend/tests/test_tool_sandbox_per_thread.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Every conversation runs its tools in its own sandbox directory. + +Parallel chats lean on this: two conversations can be mid tool call at the same +time, so a shared working directory would let one overwrite the other's files. +The session id is the chat's thread id (or project- for project chats), and +the dir is derived from it here. + +HOME is redirected at import time, so nothing touches the real ~/studio_sandbox. +""" + +import os +import sys + +import pytest + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + + +@pytest.fixture +def workdir(tmp_path, monkeypatch): + """_get_workdir with HOME pointed at tmp_path and its cache cleared.""" + from core.inference import tools + + monkeypatch.setattr(os.path, "expanduser", lambda path: str(tmp_path)) + monkeypatch.setattr(tools, "_workdirs", {}) + return tools._get_workdir + + +def test_two_conversations_get_two_directories(workdir, tmp_path): + a = workdir("thread-alpha") + b = workdir("thread-beta") + assert a != b + assert os.path.basename(a) == "thread-alpha" + assert os.path.basename(b) == "thread-beta" + assert os.path.isdir(a) and os.path.isdir(b) + assert os.path.dirname(a) == os.path.dirname(b) == str(tmp_path / "studio_sandbox") + + +def test_the_same_conversation_keeps_its_directory(workdir): + # A later turn, or a tool continuation, must land back in the same place. + assert workdir("thread-alpha") == workdir("thread-alpha") + + +def test_a_directory_is_private_to_its_conversation(workdir): + a = workdir("thread-alpha") + b = workdir("thread-beta") + with open(os.path.join(a, "secret.txt"), "w", encoding = "utf-8") as f: + f.write("alpha") + assert os.listdir(b) == [] + + +def test_project_chats_deliberately_share_one_workspace(workdir, monkeypatch): + # Chats in a project are meant to see each other's files. + from core.inference import tools + monkeypatch.setattr(tools, "_get_project_workdir", lambda sid: "/tmp/project-ws") + assert tools._get_workdir("project-abc") == "/tmp/project-ws" + + +@pytest.mark.parametrize( + "session_id", + ["../escape", "a/b", "", " ", "x" * 65], +) +def test_a_session_id_cannot_escape_the_sandbox_root(workdir, tmp_path, session_id): + resolved = workdir(session_id) if session_id else workdir(None) + root = os.path.realpath(str(tmp_path / "studio_sandbox")) + assert os.path.realpath(resolved).startswith(root + os.sep) + assert os.path.basename(resolved) in {"_invalid", "_default"} + + +def test_no_session_id_falls_back_to_default(workdir): + assert os.path.basename(workdir(None)) == "_default" + + +@pytest.mark.skipif(sys.platform == "win32", reason = "POSIX permission bits") +def test_directories_are_private_to_the_user(workdir): + assert os.stat(workdir("thread-alpha")).st_mode & 0o777 == 0o700 diff --git a/studio/backend/tests/test_tool_xml_strip.py b/studio/backend/tests/test_tool_xml_strip.py index 941d9d044a..d02638f589 100644 --- a/studio/backend/tests/test_tool_xml_strip.py +++ b/studio/backend/tests/test_tool_xml_strip.py @@ -668,7 +668,8 @@ def test_route_history_and_passthrough_forward_the_display_gate(): blocks = { "safetensors history": r"Strip stale tool-call XML from prior assistant turns.*?\.strip\(\)", "anthropic history": r"Strip stale tool-call XML via the protected display helper.*?\.strip\(\)", - "anthropic passthrough": r"gated on the declared tools so an\n.*?\.strip\(\)", + # Anchored on the code, not the comment above it, so rewrapping prose cannot break this. + "anthropic passthrough": r"if not healing_active:.*?\.strip\(\)", } for label, pat in blocks.items(): m = _re.search(pat, _src, _re.DOTALL) diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 7137fd6f96..9e72135bdd 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -12,6 +12,7 @@ import { import { ChatPage, clearNewChatDraft, + StopRunningChatsDialog, useChatRuntimeStore, type ChatSearch, } from "@/features/chat"; @@ -227,6 +228,8 @@ function RootLayout() { + {/* At the root, not under /chat: a swap can start from the Hub too. */} + {hideNavbar ? (
}> diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index a31d9b6ced..8aa4db99f4 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -520,15 +520,46 @@ export function AppSidebar() { }); const storeThreadId = useChatRuntimeStore((s) => s.activeThreadId); const setActiveThreadId = useChatRuntimeStore((s) => s.setActiveThreadId); - const anyChatRunning = useChatRuntimeStore((s) => - Object.values(s.runningByThreadId).some(Boolean), - ); - // The thread currently generating (if any), so "Return to Chat" lands on the - // live chat rather than an empty new-chat draft left active after New Chat. - const runningThreadId = useChatRuntimeStore((s) => { - const entry = Object.entries(s.runningByThreadId).find(([, on]) => on); - return entry ? entry[0] : null; - }); + // The whole map, so each row can show its own spinner. + const runningThreadIds = useChatRuntimeStore((s) => s.runningByThreadId); + // Rows, not raw thread ids: a compare conversation runs two pane threads but is one chat + // in the sidebar, so counting the map said "2 Chats" for a single compare row. + const runningChatCount = useMemo(() => { + const running = new Set( + Object.entries(runningThreadIds) + .filter(([, on]) => on) + .map(([id]) => id), + ); + if (running.size === 0) return 0; + let rows = 0; + for (const item of allChatItems) { + const ids = item.type === "compare" ? (item.threadIds ?? []) : [item.id]; + let claimed = false; + for (const id of ids) { + if (running.delete(id)) claimed = true; + } + if (claimed) rows += 1; + } + // Anything left belongs to no known row (a first turn mid-persist); count it as one. + return rows + running.size; + }, [runningThreadIds, allChatItems]); + const anyChatRunning = runningChatCount > 0; + // Where "Return to Chat" lands: the newest running chat, not the empty draft New Chat left + // active (map insertion order is start order). A compare row runs pane threads that /chat + // cannot address, so resolve those back to the pair id the route expects. + const runningTarget = useMemo(() => { + const ids = Object.entries(runningThreadIds) + .filter(([, on]) => on) + .map(([id]) => id); + const id = ids.length > 0 ? ids[ids.length - 1] : null; + if (!id) return null; + const pair = allChatItems.find( + (item) => item.type === "compare" && (item.threadIds ?? []).includes(id), + ); + return pair + ? { id: pair.id, compare: true as const } + : { id, compare: false as const }; + }, [runningThreadIds, allChatItems]); const activeThreadId = isChatRoute ? (search.thread as string | undefined) ?? (search.compare as string | undefined) ?? @@ -892,6 +923,12 @@ export function AppSidebar() { variant: "project" | "recent", ) { const isPinned = pinnedIdSet.has(item.id); + // A compare row's id is the pair id while runningByThreadId is keyed per pane thread, + // so aggregate its member threads instead. + const isGenerating = + item.type === "compare" + ? (item.threadIds ?? []).some((id) => Boolean(runningThreadIds[id])) + : Boolean(runningThreadIds[item.id]); const itemClass = variant === "project" ? "group/project-chat-item relative" @@ -951,6 +988,8 @@ export function AppSidebar() { data-testid="recent-thread" data-thread-type={item.type} data-thread-id={item.id} + data-generating={isGenerating ? "true" : undefined} + aria-busy={isGenerating || undefined} isActive={activeThreadId === item.id} className={buttonClass} onClick={() => { @@ -976,6 +1015,14 @@ export function AppSidebar() { {pendingRename?.id === item.id ? pendingRename.title : item.title} + {isGenerating && ( + + )} {variant === "project" && ( + ); +} + +function DownloadBtn({ code, name }: { code: string; name: string }) { + const download = useCallback(() => { + if (typeof document === "undefined") { + return; + } + try { + const blob = new Blob([code], { type: "text/plain;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = name; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + // Revoke next tick, after the click consumes the URL. + setTimeout(() => URL.revokeObjectURL(url), 0); + } catch { + // Never break the transcript over a download. + } + }, [code, name]); + + return ( + + ); +} + +/** A fence longer than any backtick run in the code, so a script containing ``` cannot end it early. */ +function fenceFor(source: string): string { + const longest = (source.match(/`+/g) ?? []).reduce( + (max, run) => Math.max(max, run.length), + 0, + ); + return "`".repeat(Math.max(3, longest + 1)); +} + +/** Syntax-highlighted code via Streamdown + shiki. Always in the DOM as plain monospace, but + * shiki only tokenizes once the block nears the viewport, so a long transcript does not + * highlight every script up front. Immediate where IntersectionObserver is missing. */ +function HighlightedCode({ + code: source, + language, + plain = false, +}: { + code: string; + language: string; + plain?: boolean; +}) { + const markdown = useMemo(() => { + const fence = fenceFor(source); + return `${fence}${language}\n${source}\n${fence}`; + }, [source, language]); + const containerRef = useRef(null); + const [nearViewport, setNearViewport] = useState( + () => typeof IntersectionObserver === "undefined", + ); + // Pinned to the bottom until the reader scrolls up, so a streaming payload visibly grows. + const pinnedToBottom = useRef(true); + useEffect(() => { + if (nearViewport) return; + const el = containerRef.current; + if (!el) return; + const io = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + setNearViewport(true); + io.disconnect(); + } + }, + // Highlight just before the block enters view, so it is ready on arrival. + { rootMargin: "200px" }, + ); + io.observe(el); + return () => io.disconnect(); + }, [nearViewport]); + + useEffect(() => { + const el = containerRef.current; + if (plain && el && pinnedToBottom.current) { + el.scrollTop = el.scrollHeight; + } + }, [plain, source]); + + const handleScroll = () => { + const el = containerRef.current; + if (el) { + pinnedToBottom.current = + el.scrollHeight - el.scrollTop - el.clientHeight < PIN_SLACK_PX; + } + }; + + // Skip shiki while the model is writing (it re-tokenizes every fragment) and on payloads too big. + const highlight = + nearViewport && !plain && source.length <= MAX_HIGHLIGHT_CHARS; + + return ( +
+ {highlight ? ( + + {markdown} + + ) : ( + // A div, not a
: the container's [&_pre]:!p-0 would strip the padding and shift
+        // the content when shiki swaps in. whitespace-pre so long lines scroll.
+        
+ {source} +
+ )} +
+ ); +} + +/** The code a tool is about to run, in the card's collapsible content so the chevron hides code and output together. */ +export function ToolCodeCell({ + label, + code, + language, + downloadName, + streaming = false, +}: { + label: string; + code: string; + language: string; + downloadName: string; + streaming?: boolean; +}) { + return ( +
+
+ + {label} + +
+ + +
+
+ +
+ ); +} diff --git a/studio/frontend/src/components/assistant-ui/tool-group.tsx b/studio/frontend/src/components/assistant-ui/tool-group.tsx index 469c449a70..af370d892e 100644 --- a/studio/frontend/src/components/assistant-ui/tool-group.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-group.tsx @@ -10,7 +10,11 @@ import { } from "react"; import { useAuiState } from "@assistant-ui/react"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; -import { toolOutputKey, useToolPaneScope } from "@/features/chat"; +import { + toolOutputKey, + useToolPaneScope, + useUnresolvedToolPaneScope, +} from "@/features/chat"; import { ChevronDownIcon } from "lucide-react"; import { Wrench01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; @@ -239,16 +243,23 @@ const ToolGroupImpl: FC< // Force the group open when any call is receiving tool_output events. const toolLiveOutput = useChatRuntimeStore((s) => s.toolLiveOutput); const paneScope = useToolPaneScope(); + const unresolvedScope = useUnresolvedToolPaneScope(); const hasLiveOutput = useAuiState(({ message }) => message.parts .slice(startIndex, endIndex + 1) .some( (part) => part.type === "tool-call" && - Object.prototype.hasOwnProperty.call( + // Either scope: a first turn writes under the unresolved one for its whole + // life, even after the autosave assigns the id (see useToolOutputFor). + (Object.prototype.hasOwnProperty.call( toolLiveOutput, toolOutputKey(paneScope, part.toolCallId), - ), + ) || + Object.prototype.hasOwnProperty.call( + toolLiveOutput, + toolOutputKey(unresolvedScope, part.toolCallId), + )), ), ); // Keep the group open once a confirmation or live output forced it (so an diff --git a/studio/frontend/src/components/assistant-ui/tool-live-output.tsx b/studio/frontend/src/components/assistant-ui/tool-live-output.tsx index 3434783f0a..df202b57f0 100644 --- a/studio/frontend/src/components/assistant-ui/tool-live-output.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-live-output.tsx @@ -4,7 +4,7 @@ "use client"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; -import { toolOutputKey, useToolPaneScope } from "@/features/chat"; +import { useToolOutputFor, useToolPaneScope } from "@/features/chat"; import { useEffect, useMemo, useRef } from "react"; import { tailText } from "./tool-result-output"; @@ -16,8 +16,10 @@ import { tailText } from "./tool-result-output"; */ export function ToolLiveOutput({ toolCallId }: { toolCallId: string }) { const paneScope = useToolPaneScope(); - const output = useChatRuntimeStore( - (s) => s.toolLiveOutput[toolOutputKey(paneScope, toolCallId)] ?? "", + const output = useToolOutputFor( + useChatRuntimeStore((s) => s.toolLiveOutput), + paneScope, + toolCallId, ); const scrollRef = useRef(null); // Pinned to the bottom until the user scrolls up (handler below), so diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx index 34e51b9d5a..e058a04ed1 100644 --- a/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx @@ -3,28 +3,25 @@ "use client"; -import { copyToClipboard } from "@/lib/copy-to-clipboard"; import { getAuthToken } from "@/features/auth/session"; import type { ToolCallMessagePartComponent } from "@assistant-ui/react"; import { useToolArgsStatus } from "@assistant-ui/react"; -import { code as codePlugin } from "@streamdown/code"; -import { CodeIcon, CopyIcon, DownloadIcon } from "lucide-react"; -import { Tick02Icon } from "@/lib/tick-icon"; -import { HugeiconsIcon } from "@hugeicons/react"; +import { CodeIcon } from "lucide-react"; import { Spinner } from "@/components/ui/spinner"; -import { memo, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { Streamdown } from "streamdown"; +import { memo } from "react"; import { ToolFallbackContent, ToolFallbackRoot, ToolFallbackTrigger, } from "./tool-fallback"; +import { CopyBtn, ToolCodeCell } from "./tool-code-cell"; import { ToolLiveOutput } from "./tool-live-output"; import { ToolResultOutput } from "./tool-result-output"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; import { preferFullToolOutput, - toolOutputKey, + useToolAwaitingApproval, + useToolOutputFor, useToolPaneScope, } from "@/features/chat"; @@ -34,151 +31,6 @@ interface StructuredResult { sessionId: string; } -const MAX_DISPLAY = 10_000; -const COPY_RESET_MS = 2000; -const SHIKI_THEME = ["github-light", "github-dark"] as ["github-light", "github-dark"]; - -function truncate(text: string): string { - return text.length <= MAX_DISPLAY - ? text - : `${text.slice(0, MAX_DISPLAY)}\n... (truncated)`; -} - -function CopyBtn({ text }: { text: string }) { - const [copied, setCopied] = useState(false); - const timer = useRef | null>(null); - - useEffect(() => { - return () => { - if (timer.current) { - clearTimeout(timer.current); - } - }; - }, []); - - const copy = useCallback(async () => { - if (await copyToClipboard(text)) { - setCopied(true); - if (timer.current) { - clearTimeout(timer.current); - } - timer.current = setTimeout(() => setCopied(false), COPY_RESET_MS); - } - }, [text]); - - return ( - - ); -} - -/** Save the script as a .py file via a client-side Blob. */ -function DownloadBtn({ code, name = "script.py" }: { code: string; name?: string }) { - const download = useCallback(() => { - if (typeof document === "undefined") { - return; - } - try { - const blob = new Blob([code], { type: "text/x-python" }); - const url = URL.createObjectURL(blob); - const anchor = document.createElement("a"); - anchor.href = url; - anchor.download = name; - document.body.appendChild(anchor); - anchor.click(); - anchor.remove(); - // Revoke next tick, after the click consumes the URL. - setTimeout(() => URL.revokeObjectURL(url), 0); - } catch { - // Best-effort: never break the transcript over a download. - } - }, [code, name]); - - return ( - - ); -} - -/** Syntax-highlighted code via Streamdown + shiki; inherits parent container. - * The script is always in the DOM (a plain monospace placeholder), but shiki - * only tokenizes once the block scrolls near the viewport, so a long transcript - * with many scripts doesn't highlight every one up front. Falls back to - * immediate highlight when IntersectionObserver is unavailable (SSR / tests). */ -function HighlightedCode({ code: source, language }: { code: string; language: string }) { - const display = useMemo(() => truncate(source), [source]); - const markdown = useMemo( - () => `\`\`\`${language}\n${display}\n\`\`\``, - [display, language], - ); - const containerRef = useRef(null); - const [highlight, setHighlight] = useState( - () => typeof IntersectionObserver === "undefined", - ); - useEffect(() => { - if (highlight) return; - const el = containerRef.current; - if (!el) return; - const io = new IntersectionObserver( - (entries) => { - if (entries.some((entry) => entry.isIntersecting)) { - setHighlight(true); - io.disconnect(); - } - }, - // Highlight just before the block enters view so it's colorized by the - // time the user reaches it, without tokenizing off-screen scripts. - { rootMargin: "200px" }, - ); - io.observe(el); - return () => io.disconnect(); - }, [highlight]); - return ( -
- {highlight ? ( - - {markdown} - - ) : ( - // A div, not a
: the container's [&_pre]:!p-0 would override a
-        // 
's padding and shift the content by p-3 when shiki swaps in. Keep
-        // the same p-3, and whitespace-pre (not pre-wrap) so long lines scroll in
-        // the container's overflow-auto exactly like the highlighted 
, rather
-        // than wrapping taller and then collapsing when shiki swaps in.
-        
- {display} -
- )} -
- ); -} - function isStructuredResult(val: unknown): val is StructuredResult { return ( typeof val === "object" && @@ -221,46 +73,50 @@ const PythonToolUIImpl: ToolCallMessagePartComponent = ({ // Show the fuller live stream over a truncated result, keeping its exit // status. Session-transient: after a reload only the result remains. const paneScope = useToolPaneScope(); - const fullOutput = useChatRuntimeStore( - (s) => s.toolFullOutput[toolOutputKey(paneScope, toolCallId)] ?? "", + const fullOutput = useToolOutputFor( + useChatRuntimeStore((s) => s.toolFullOutput), + paneScope, + toolCallId, ); const displayOutput = preferFullToolOutput(fullOutput, output); const authToken = getAuthToken(); + // The gate only opens once the call parsed, so a pending approval means the script is + // written even while the args status still reads as streaming. + const awaitingApproval = useToolAwaitingApproval(toolCallId); + const isWriting = isWritingCode && !awaitingApproval; return ( - // Status/output collapse from history; the script source renders outside - // ToolFallbackContent so it stays visible on reopen (#7165). + // Script, status and output all collapse behind the one chevron. - {code && ( -
-
-
- - script - -
- - -
-
- -
-
- )} + {code && ( + + )}
{/* Output */} {isRunning ? ( <>
- {isWritingCode ? "Writing code…" : "Running…"} + + {awaitingApproval + ? "Waiting for approval…" + : isWriting + ? "Writing code…" + : "Running…"} +
{/* Live stdout streamed via tool_output SSE events. */} diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx index 17ae26d388..b6ea2aaa6f 100644 --- a/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx @@ -3,69 +3,27 @@ "use client"; -import { copyToClipboard } from "@/lib/copy-to-clipboard"; import type { ToolCallMessagePartComponent } from "@assistant-ui/react"; import { useToolArgsStatus } from "@assistant-ui/react"; -import { CopyIcon, TerminalIcon } from "lucide-react"; -import { Tick02Icon } from "@/lib/tick-icon"; -import { HugeiconsIcon } from "@hugeicons/react"; +import { TerminalIcon } from "lucide-react"; import { Spinner } from "@/components/ui/spinner"; -import { memo, useCallback, useEffect, useRef, useState } from "react"; +import { memo } from "react"; import { ToolFallbackContent, ToolFallbackRoot, ToolFallbackTrigger, } from "./tool-fallback"; +import { CopyBtn, ToolCodeCell } from "./tool-code-cell"; import { ToolLiveOutput } from "./tool-live-output"; import { ToolResultOutput } from "./tool-result-output"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; import { preferFullToolOutput, - toolOutputKey, + useToolAwaitingApproval, + useToolOutputFor, useToolPaneScope, } from "@/features/chat"; -const COPY_RESET_MS = 2000; - -function CopyBtn({ text }: { text: string }) { - const [copied, setCopied] = useState(false); - const timer = useRef | null>(null); - - useEffect(() => { - return () => { - if (timer.current) { - clearTimeout(timer.current); - } - }; - }, []); - - const copy = useCallback(async () => { - if (await copyToClipboard(text)) { - setCopied(true); - if (timer.current) { - clearTimeout(timer.current); - } - timer.current = setTimeout(() => setCopied(false), COPY_RESET_MS); - } - }, [text]); - - return ( - - ); -} - const TerminalToolUIImpl: ToolCallMessagePartComponent = ({ toolCallId, args, @@ -87,13 +45,19 @@ const TerminalToolUIImpl: ToolCallMessagePartComponent = ({ // Show the fuller live stream over a truncated result, keeping its exit // status. Session-transient: after a reload only the result remains. const paneScope = useToolPaneScope(); - const fullOutput = useChatRuntimeStore( - (s) => s.toolFullOutput[toolOutputKey(paneScope, toolCallId)] ?? "", + const fullOutput = useToolOutputFor( + useChatRuntimeStore((s) => s.toolFullOutput), + paneScope, + toolCallId, ); const displayOutput = preferFullToolOutput(fullOutput, output); + // The gate only opens once the call parsed, so a pending approval means the command is + // written even while the args status still reads as streaming. + const awaitingApproval = useToolAwaitingApproval(toolCallId); + const isWriting = isWritingCommand && !awaitingApproval; return ( - // Open when mounted mid-run so live output shows; collapsed from history. + // Open mid-run so command and live output show, collapsed from history. + {command && ( + + )}
{isRunning ? ( <>
- {isWritingCommand ? "Writing command…" : "Running…"} + + {awaitingApproval + ? "Waiting for approval…" + : isWriting + ? "Writing command…" + : "Running…"} +
{/* Live stdout streamed via tool_output SSE events. */} diff --git a/studio/frontend/src/components/ui/spinner.tsx b/studio/frontend/src/components/ui/spinner.tsx index 283b4e21de..34543ed763 100644 --- a/studio/frontend/src/components/ui/spinner.tsx +++ b/studio/frontend/src/components/ui/spinner.tsx @@ -6,15 +6,22 @@ import { Loader2Icon } from "lucide-react"; import { cn } from "@/lib/utils"; -/** - * App-wide spinner: a clean circular arc with a rounded cap (lucide - * Loader2 / LoaderCircle), animated, inheriting the current text color. - */ -function Spinner({ className }: { className?: string }) { +/** App-wide spinner inheriting the current text color. `label` overrides the announcement + * where "loading" is not what it means (a sidebar chat is generating). */ +function Spinner({ + className, + label = "Loading", + "data-testid": dataTestId, +}: { + className?: string; + label?: string; + "data-testid"?: string; +}) { return ( ); diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index a0be3ea640..b9e7229e34 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -59,6 +59,7 @@ import { shouldPreserveFullOutput, toolOutputKey, toolPaneScope, + toolThreadScope, } from "../tool-output-scope"; import type { ModelType } from "../types"; import { isMultimodalResponse } from "../types/api"; @@ -2232,7 +2233,20 @@ export function createOpenAIStreamAdapter( : undefined; const threadKey = resolvedThreadId; - runtime.setThreadRunning(threadKey, true); + // The run is durable on the server, but Stop, archive and delete reach a background + // thread only through this map: without a handle the supervisor kept planning against + // a deleted conversation. Registered before the run exists, since the thread can be + // stopped while createResearchRun is still in flight. + let researchRunId: string | null = null; + let researchStopRequested = false; + const researchServerCancel = () => { + researchStopRequested = true; + if (researchRunId) { + void cancelResearchRun(researchRunId).catch(() => {}); + } + }; + runtime.registerThreadServerCancel(threadKey, researchServerCancel); + runtime.setThreadRunning(threadKey, true, { owner: researchServerCancel }); let report = ""; let releaseResearchFollow: (() => void) | null = null; const researchFollowController = new AbortController(); @@ -2272,6 +2286,13 @@ export function createOpenAIStreamAdapter( blockedDomains: [...runtime.researchWebsitePolicy.blockedDomains], }, }); + researchRunId = createdRun.id; + if (researchStopRequested) { + // Stopped while createResearchRun was still in flight, so the handle had no + // id to act on. Replay it rather than following a run the user already ended. + void cancelResearchRun(createdRun.id).catch(() => {}); + return; + } releaseResearchFollow = beginExternalResearchFollow( createdRun, detachResearchFollow, @@ -2330,7 +2351,8 @@ export function createOpenAIStreamAdapter( } finally { abortSignal.removeEventListener("abort", forwardAdapterAbort); releaseResearchFollow?.(); - runtime.setThreadRunning(threadKey, false); + runtime.clearThreadServerCancel(threadKey, researchServerCancel); + runtime.setThreadRunning(threadKey, false, { owner: researchServerCancel }); } return; } @@ -2339,17 +2361,21 @@ export function createOpenAIStreamAdapter( ? `${sandboxSessionId || "_default"}:${resolvedThreadId}` : sandboxSessionId || "_default"; const toolConfirmationIdsByBackendId = new Map(); - // Store keys are pane-scoped since local tool ids ("call_0") repeat across - // turns and concurrent panes (compare mode). Track this run's keys so - // cleanup can't wipe another pane's. - const toolOutputPaneScope = toolPaneScope( - options.modelType, - options.pairId, + // Local tool ids ("call_0") repeat across turns, panes and conversations, so scope by pane + // AND thread. unstable_threadId alone, no activeThreadId fallback: the reader has only + // threadListItem.remoteId, which is exactly this value. + const toolOutputPaneScope = toolThreadScope( + toolPaneScope(options.modelType, options.pairId), + unstable_threadId, ); const scopedToolOutputKey = (id: string) => toolOutputKey(toolOutputPaneScope, id); const runToolLiveOutputKeys = new Set(); const resolvedThreadKey = resolvedThreadId ?? null; + // Which conversation was on screen when this run started. A first turn has no id yet, so + // this is the only way to tell later whether the user has switched away from it. + const activeThreadIdAtRunStart = + useChatRuntimeStore.getState().activeThreadId ?? null; const pendingImageEditReferenceForRun = runtime.pendingImageEditReference; const selectedImageEditReference = (pendingImageEditReferenceForRun?.threadId ?? null) === @@ -2755,8 +2781,11 @@ export function createOpenAIStreamAdapter( // waitForRunEnd resolves instead of hanging: this gate fires // before the streaming path's setThreadRunning(true). const gatedThreadKey = resolvedThreadId || "__default"; - runtime.setThreadRunning(gatedThreadKey, true); - runtime.setThreadRunning(gatedThreadKey, false); + // Own token: siblings share "__default", so an ownerless clear would drop their + // entries while they are still generating. + const gateOwner = () => {}; + runtime.setThreadRunning(gatedThreadKey, true, { owner: gateOwner }); + runtime.setThreadRunning(gatedThreadKey, false, { owner: gateOwner }); clearSelectedImageEditReference(); throw new Error(imageGateReason); } @@ -2774,13 +2803,44 @@ export function createOpenAIStreamAdapter( } const useAdapter = await resolveUseAdapter(resolvedThreadId, options); + const threadKey = resolvedThreadId || "__default"; + // A first turn files its handles under "__default"; autosave then assigns a real id and + // adoptDefaultThreadRun re-keys them mid-run. Resolve per use so later writes and the + // final clear follow the run instead of stranding entries behind. + const liveThreadKey = (owner: () => void) => + threadKey === "__default" + ? useChatRuntimeStore.getState().runKeyForOwner(threadKey, owner) + : threadKey; + + // Per-run token so a delayed stop POST can't match the next run. + const cancelId = + typeof crypto !== "undefined" && "randomUUID" in crypto + ? crypto.randomUUID() + : `${Date.now()}-${Math.random().toString(36).slice(2)}`; + + // Per-run abort, chained to assistant-ui's signal. cancelByThreadId only holds the visible + // thread's cancelRun(), so this controller is the only way to end a backgrounded chat's + // request; the cancel POST below reaches llama-server only. + const runAbort = new AbortController(); + const runSignal = runAbort.signal; + const forwardAbort = () => runAbort.abort(abortSignal.reason); + // Declared here, not at its registration below: it doubles as this run's identity token + // on the per-thread maps (see registerThreadServerCancel). + const serverCancel = () => runAbort.abort(); + if (abortSignal.aborted) { + forwardAbort(); + } else { + abortSignal.addEventListener("abort", forwardAbort, { once: true }); + } + // ── Audio model path (non-streaming) ───────────────────── const activeModel = runtime.models.find( (m) => m.id === params.checkpoint, ); if (activeModel?.isAudio && !activeModel?.hasAudioInput) { - const threadKey = resolvedThreadId || "__default"; - runtime.setThreadRunning(threadKey, true); + const audioCancel = () => runAbort.abort(); + runtime.registerThreadServerCancel(threadKey, audioCancel); + runtime.setThreadRunning(threadKey, true, { owner: audioCancel }); try { yield { content: [{ type: "text" as const, text: "Generating audio..." }], @@ -2790,6 +2850,10 @@ export function createOpenAIStreamAdapter( { model: params.checkpoint, messages: outboundMessages, + // Same run in both registries: without it the backend files this under no + // thread, and the stop-chats prompt counts the named local run and the + // unnamed backend one as two. + ...(resolvedThreadId ? { thread_id: resolvedThreadId } : {}), stream: false, temperature: params.temperature, top_p: params.topP, @@ -2800,7 +2864,7 @@ export function createOpenAIStreamAdapter( presence_penalty: params.presencePenalty, ...(useAdapter === undefined ? {} : { use_adapter: useAdapter }), }, - abortSignal, + runSignal, ); const audioUrl = `data:audio/wav;base64,${result.audio.data}`; @@ -2813,19 +2877,21 @@ export function createOpenAIStreamAdapter( ], }; } catch (err) { - if (!abortSignal.aborted) { + if (!runSignal.aborted) { toast.error("Audio generation failed", { description: err instanceof Error ? err.message : "Unknown error", }); } throw err; } finally { - runtime.setThreadRunning(threadKey, false); + abortSignal.removeEventListener("abort", forwardAbort); + const audioKey = liveThreadKey(audioCancel); + runtime.setThreadRunning(audioKey, false, { owner: audioCancel }); + runtime.clearThreadServerCancel(audioKey, audioCancel); } return; } - const threadKey = resolvedThreadId || "__default"; let waitingFirstChunk = true; let firstTokenSettled = false; const streamStartTime = Date.now(); @@ -2856,10 +2922,15 @@ export function createOpenAIStreamAdapter( const warmupDelayMs = 450; const warmupTimer = setTimeout(() => { if (!waitingFirstChunk) return; - if (abortSignal.aborted) return; + if (runSignal.aborted) return; runtime.setGeneratingStatus("waiting"); }, warmupDelayMs); - runtime.setThreadRunning(threadKey, true); + // Flagged local/external so the model-swap gate only counts the chats a reload ends; the + // backend leaves external-provider runs out of active_generations for the same reason. + runtime.setThreadRunning(threadKey, true, { + local: !isExternalRequest, + owner: serverCancel, + }); let cumulativeText = ""; let reasoningStartAt: number | null = null; let reasoningDuration = 0; @@ -3025,21 +3096,12 @@ export function createOpenAIStreamAdapter( timings?: ServerTimings; } | null = null; - // Per-run cancellation token so a delayed stop POST can't match - // the next run on the same thread. - const cancelId = - typeof crypto !== "undefined" && "randomUUID" in crypto - ? crypto.randomUUID() - : `${Date.now()}-${Math.random().toString(36).slice(2)}`; - // Colab-style proxies can swallow fetch aborts, so also POST // /inference/cancel explicitly on abort. const onAbortCancel = () => { - // assistant-ui aborts with AbortError(detach=true) when a thread's runtime - // unmounts (navigation / background thread switch) and detach=false for an - // explicit Stop. Only a real Stop cancels the backend run; a detach must - // leave a backgrounded generation streaming. - if ((abortSignal.reason as { detach?: boolean } | undefined)?.detach) { + // assistant-ui aborts with detach=true when a runtime unmounts and detach=false for an + // explicit Stop. Only a real Stop cancels the backend run; runSignal forwards the reason. + if ((runSignal.reason as { detach?: boolean } | undefined)?.detach) { return; } const body: Record = { cancel_id: cancelId }; @@ -3060,11 +3122,17 @@ export function createOpenAIStreamAdapter( keepalive: true, }).catch(() => {}); }; + + // Stop handle for when this conversation is not the visible one, which cancelByThreadId + // cannot reach. Aborting this run's own controller closes just its request, and the + // listener above posts its cancel_id so llama-server stops decoding too. For an + // external provider the abort is the stop, since its cancel_id is never registered. + runtime.registerThreadServerCancel(threadKey, serverCancel); try { - if (abortSignal.aborted) { + if (runSignal.aborted) { onAbortCancel(); } else { - abortSignal.addEventListener("abort", onAbortCancel, { once: true }); + runSignal.addEventListener("abort", onAbortCancel, { once: true }); } const { @@ -3536,7 +3604,7 @@ export function createOpenAIStreamAdapter( } clearSelectedImageEditReference(); await ThreadAutosaveHandle.awaitFirstSave(resolvedThreadId); - const stream = streamChatCompletions(requestPayload, abortSignal); + const stream = streamChatCompletions(requestPayload, runSignal); for await (const chunk of stream) { const chunkModel = (chunk as { model?: unknown }).model; @@ -3549,7 +3617,11 @@ export function createOpenAIStreamAdapter( chunk as unknown as { _toolStatus?: string } )._toolStatus; if (toolStatusText !== undefined) { - runtime.setToolStatus(toolStatusText || null); + runtime.setToolStatus( + liveThreadKey(serverCancel), + toolStatusText || null, + serverCancel, + ); continue; } @@ -3578,7 +3650,9 @@ export function createOpenAIStreamAdapter( } )._diffusionFrame; if (diffusionFrame !== undefined) { - runtime.setActiveDiffusionCanvas({ + // Keyed by thread so a background run's frames stay out of the visible chat + // instead of overwriting the frame it is painting. + runtime.setActiveDiffusionCanvas(liveThreadKey(serverCancel), { block: diffusionFrame.block ?? 0, step: diffusionFrame.step ?? 0, total: diffusionFrame.total ?? 0, @@ -3719,8 +3793,16 @@ export function createOpenAIStreamAdapter( const approvalId = (toolEvent.approval_id as string) || ""; const awaitingConfirmation = toolEvent.awaiting_confirmation === true; + // Reuse a provisional card's part id, else the confirmation-scoped id + // opens a second card and the first spins "Running" forever. + const openPartId = backendToolCallId + ? toolPartIdByBackendId.get(backendToolCallId) + : undefined; + const reuseOpenPart = + !!openPartId && + toolCallParts.some((p) => p.toolCallId === openPartId); const id = - awaitingConfirmation && approvalId + awaitingConfirmation && approvalId && !reuseOpenPart ? `${toolConfirmationScopeId}:${approvalId}` : backendToolCallId ? resolveToolPartId(backendToolCallId) @@ -4299,9 +4381,17 @@ export function createOpenAIStreamAdapter( // Anthropic-only (billed at the write premium). const cacheWriteTokens = meta?.usage?.cache_creation_input_tokens ?? 0; - // Gate on the captured checkpoint still being active so a late - // completion from provider A doesn't populate the bar after a - // mid-stream switch to provider B. + // Gate on the captured checkpoint so a late completion from provider A cannot populate + // the bar after a mid-stream switch to B, and on the captured thread so a background + // run finishing after New Chat cannot repaint another chat's usage. An unresolved run + // has no id to compare, so compare what was on screen when it started. A first turn is + // adopted onto an id mid-run and autosave moves activeThreadId with it, so read the + // adopted key, or the run stays "unresolved" for life and the bar stays blank. + const usageKey = liveThreadKey(serverCancel); + const usageThreadKey = usageKey === "__default" ? null : usageKey; + const usageThreadIsVisible = + useChatRuntimeStore.getState().activeThreadId === + (usageThreadKey ?? activeThreadIdAtRunStart); if ( meta?.usage && typeof meta.usage.prompt_tokens === "number" && @@ -4309,13 +4399,23 @@ export function createOpenAIStreamAdapter( typeof meta.usage.total_tokens === "number" && useChatRuntimeStore.getState().params.checkpoint === params.checkpoint ) { - useChatRuntimeStore.getState().setContextUsage({ + const usage = { promptTokens: meta.usage.prompt_tokens, completionTokens: meta.usage.completion_tokens, totalTokens: meta.usage.total_tokens, cachedTokens, cacheWriteTokens, - }); + }; + // File it under this run's own thread even when the gate below blocks the visible + // write, so switching back re-applies it. + if (usageThreadKey !== null) { + useChatRuntimeStore + .getState() + .setThreadContextUsage(usageThreadKey, usage); + } + if (usageThreadIsVisible) { + useChatRuntimeStore.getState().setContextUsage(usage); + } } const finishedAt = Date.now(); @@ -4368,7 +4468,7 @@ export function createOpenAIStreamAdapter( settleFirstTokenErr( err instanceof Error ? err : new Error("Generation failed"), ); - if (!abortSignal.aborted) { + if (!runSignal.aborted) { const msg = err instanceof Error ? err.message : String(err); if (err instanceof GenerationLengthError) { toast.error("Response ran out of tokens", { @@ -4406,13 +4506,18 @@ export function createOpenAIStreamAdapter( } throw err; } finally { - abortSignal.removeEventListener("abort", onAbortCancel); + runSignal.removeEventListener("abort", onAbortCancel); + abortSignal.removeEventListener("abort", forwardAbort); + // Resolve once: the clears below drop the owner the lookup keys on. + const cleanupKey = liveThreadKey(serverCancel); const confirmStore = useChatRuntimeStore.getState(); for (const part of toolCallParts) { confirmStore.clearToolConfirmation(part.toolCallId); } runtime.setGeneratingStatus(null); - runtime.setToolStatus(null); + // Scoped by thread AND by run: a global clear wiped every other running chat's badge, + // and an unowned one wiped a concurrent run's badge behind the same key. + runtime.setToolStatus(cleanupKey, null, serverCancel); // Clear only this run's live keys (a concurrent pane owns its own). A // key still here streamed stdout but never reached tool_end (SSE drop or // cancel), so promote it to full output first, else the partial @@ -4426,20 +4531,23 @@ export function createOpenAIStreamAdapter( store.clearToolLiveOutput(liveKey); } runToolLiveOutputKeys.clear(); - // Drop the transient denoising canvas so the finished bubble shows only - // the committed markdown answer (cancellation/error included). - runtime.setActiveDiffusionCanvas(null); + // Drop the transient denoising canvas so the finished bubble shows only the committed + // answer. Scoped: a global clear wiped another denoising chat's frame. + runtime.clearActiveDiffusionCanvasForThread(cleanupKey); clearTimeout(warmupTimer); if (waitingFirstChunk) { if (firstTokenSettled) { settleFirstTokenOk(); - } else if (abortSignal.aborted) { + } else if (runSignal.aborted) { settleFirstTokenErr(new Error("Cancelled")); } else { settleFirstTokenErr(new Error("No tokens received")); } } - runtime.setThreadRunning(threadKey, false); + // serverCancel narrows both clears: runs with no resolved thread id share the "__default" + // key, so a blind clear could drop a sibling's entry. + runtime.setThreadRunning(cleanupKey, false, { owner: serverCancel }); + runtime.clearThreadServerCancel(cleanupKey, serverCancel); } }, }; diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 4f558545ca..a40867beea 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -129,6 +129,27 @@ export async function getApiMonitorEntry(id: string): Promise { return parseJsonOrThrow(response); } +export interface ActiveGenerationsResponse { + count: number; + /** Conversations with a generation in flight. Shorter than `count` when a + * first turn started before its thread id was persisted. */ + thread_ids: string[]; + /** One entry per in-flight request. `kind` is "chat" unless it is an + * embeddings / completions / audio call, which has no conversation. */ + active?: { thread_id: string | null; kind?: string }[]; + parallel_slots: number; +} + +/** + * Chats generating on the backend right now. Authoritative where `runningByThreadId` is not: + * that map is per-tab, empty after a reload and blind to a second tab, and /load and /unload + * 409 on these. + */ +export async function getActiveGenerations(): Promise { + const response = await authFetch("/api/inference/active-generations"); + return parseJsonOrThrow(response); +} + export async function loadModel( payload: LoadModelRequest, ): Promise { diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 7cc03fab26..8ae6c6fb15 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -2682,9 +2682,10 @@ export function ChatPage({ ggufNativeContextLength: null, activeNativePathToken: null, activeNativePathExpiresAtMs: null, - // Clear previous-model counters, else the relaxed external-provider - // render gate shows stale stats until the next completion. + // Clear previous-model counters, else the relaxed external-provider render gate shows + // stale stats. The per-thread copies go too, so a switch back cannot re-apply. contextUsage: null, + contextUsageByThreadId: {}, supportsReasoning: reasoningCaps.supportsReasoning, reasoningAlwaysOn: reasoningCaps.reasoningAlwaysOn, reasoningStyle: reasoningCaps.reasoningStyle, @@ -2906,7 +2907,13 @@ export function ChatPage({ ) { return; } - store.setContextUsage(usage); + // Key by the thread this restore read, like the history loader: the await above can + // outlast a switch away, and an unkeyed write would file this thread's usage under + // the incoming one. + store.setThreadContextUsage(threadId, usage); + if (store.activeThreadId === threadId) { + store.setContextUsage(usage); + } }) .catch((error) => { if (!isExpectedBackgroundChatStorageError(error)) { diff --git a/studio/frontend/src/features/chat/components/stop-running-chats-dialog.tsx b/studio/frontend/src/features/chat/components/stop-running-chats-dialog.tsx new file mode 100644 index 0000000000..dd9d6c13a1 --- /dev/null +++ b/studio/frontend/src/features/chat/components/stop-running-chats-dialog.tsx @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { useStopRunningChatsDialogStore } from "../stores/stop-running-chats-dialog-store"; + +/** + * Confirmation for applying a model or reload-required setting while chats are generating. + * They share one llama-server, so the swap ends all of them: name them and make the user + * opt in rather than truncating silently. + */ +export function StopRunningChatsDialog() { + const open = useStopRunningChatsDialogStore((s) => s.open); + const count = useStopRunningChatsDialogStore((s) => s.count); + const titles = useStopRunningChatsDialogStore((s) => s.titles); + const action = useStopRunningChatsDialogStore((s) => s.action); + const hasNonChat = useStopRunningChatsDialogStore((s) => s.hasNonChat); + const effect = useStopRunningChatsDialogStore((s) => s.effect); + const resolve = useStopRunningChatsDialogStore((s) => s.resolve); + + // Embeddings, raw completions and audio share the model but are not conversations, + // so name them generically rather than offering to stop chats that do not exist. + const noun = hasNonChat + ? count === 1 + ? "request" + : "requests" + : count === 1 + ? "chat" + : "chats"; + const sharer = hasNonChat ? "request" : "conversation"; + // Ejecting leaves no model loaded. Saying it "reloads the model" and offering "Stop and + // reload" promised the opposite of what confirming does, for the destructive one. + const unloads = effect === "unload"; + const lead = unloads + ? `${action || "Unloading the model"} leaves no model loaded, and every open ${sharer} shares it, ` + : `${action ? `${action} reloads the model, ` : "Reloading the model "}which every open ${sharer} shares, `; + const shown = titles.slice(0, 5); + const remaining = Math.max(0, titles.length - shown.length); + + return ( + { + // Escape / overlay click must resolve, or the caller's await hangs. + if (!next) resolve(false); + }} + > + + + + Stop {count} running {noun}? + + + {lead}so {count === 1 ? "this" : "these"} {noun} will stop + {hasNonChat ? "" : " generating"}. Work produced so far is kept. + + + {shown.length > 0 && ( +
    + {shown.map((title) => ( +
  • + {title} +
  • + ))} + {remaining > 0 && ( +
  • + and {remaining} more +
  • + )} +
+ )} + + resolve(false)}> + Keep generating + + resolve(true)}> + {unloads ? "Stop and unload" : "Stop and reload"} + + +
+
+ ); +} diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 48a6168555..d4057591b0 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -28,6 +28,7 @@ import { validateModel, } from "../api/chat-api"; import { formatEta, formatRate } from "../utils/format-transfer"; +import { confirmStopRunningChatsIfNeeded } from "../utils/confirm-stop-running-chats"; import { GPU_LAYERS_AUTO, isLocalModelPath, @@ -463,7 +464,14 @@ export function useChatModelRuntime() { useChatRuntimeStore.getState().setModelLoading(true); void (async () => { try { + // Unforced on purpose: a chat may stream on the PREVIOUS model and must not be killed by + // cancelling this load. Nothing to report, since the route runs its stop-loading fast + // path ahead of the active-chat refusal. await unloadModel({ model_path: model.id }).catch(() => {}); + // clearCheckpoint above assumed nothing was left loaded, but a forced switch keeps the + // previous model resident until /load's teardown, and the stop-loading fast path leaves + // it there. Take the answer from the backend, which reports none once it was evicted. + await syncInferenceStatusToStore().catch(() => {}); } finally { cancelUnloadPendingRef.current = false; if (!loadingModelRef.current) { @@ -505,10 +513,11 @@ export function useChatModelRuntime() { // as a duplicate), don't start a second concurrent load and don't swallow the // request: surface it so the user waits or cancels. Centralized here so every // entry point is covered, not just the staged Load button. - const inFlightLoad = - loadingModelRef.current ?? - useChatRuntimeStore.getState().loadingModelPick; - if (inFlightLoad) { + const bailIfLoadInFlight = (): boolean => { + const inFlightLoad = + loadingModelRef.current ?? + useChatRuntimeStore.getState().loadingModelPick; + if (!inFlightLoad) return false; if (typeof selection !== "string" && selection.previousConfig) { applyPerModelConfigToRuntime(selection.previousConfig); } @@ -516,7 +525,7 @@ export function useChatModelRuntime() { inFlightLoad.id === modelId && (inFlightLoad.ggufVariant ?? null) === (ggufVariant ?? null) && (inFlightLoad.nativePathToken ?? null) === (nativePathToken ?? null); - if (loadingSamePick) return; + if (loadingSamePick) return true; const message = "Another model is already loading. Wait for it to finish or cancel it first."; setModelsError(message); @@ -524,8 +533,61 @@ export function useChatModelRuntime() { toast.info("Another model is already loading", { description: "Wait for it to finish or cancel it first.", }); + return true; + }; + if (bailIfLoadInFlight()) return; + + // Picking an external provider leaves the local model resident and stops the status poll + // mirroring it, so params.checkpoint cannot tell whether this pick is that same model. + // Ask the backend before prompting: /load answers already_loaded ahead of its cancel + // hook, so the dialog would promise to stop chats this pick never interrupts. A staged + // config always carries forceReload, so Apply still reloads and prompts. + const selectedCheckpoint = + useChatRuntimeStore.getState().params.checkpoint; + if (!forceReload && isExternalModelId(selectedCheckpoint)) { + const residentStatus = await getInferenceStatus().catch(() => null); + if ( + residentStatus && + resolveInferenceCheckpointId(residentStatus) === modelId && + (residentStatus.gguf_variant ?? null) === (ggufVariant ?? null) + ) { + // Same window as the confirm below: a rival load may have started during that GET, + // and it owns the resident model now. + if (bailIfLoadInFlight()) return; + // Roll back the config pre-applied for the load that is not happening BEFORE hydrating, + // so the resident model's status wins over the staged snapshot. + if (typeof selection !== "string" && selection.previousConfig) { + applyPerModelConfigToRuntime(selection.previousConfig); + } + const previousGgufVariant = + useChatRuntimeStore.getState().activeGgufVariant; + useChatRuntimeStore + .getState() + .setCheckpoint(modelId, residentStatus.gguf_variant); + applyActiveModelStatusToStore(residentStatus, { + previousCheckpoint: selectedCheckpoint, + previousGgufVariant, + }); + syncModelCapabilities(modelId, residentStatus); + return; + } + } + + // Every chat decodes on the llama-server this load replaces, so ask first, then allow the + // cancel; the 409 gate stays armed for callers that never confirmed. + const stopDecision = await confirmStopRunningChatsIfNeeded( + forceReload ? "Applying these settings" : "Loading a different model", + ); + if (!stopDecision.proceed) { + if (typeof selection !== "string" && selection.previousConfig) { + applyPerModelConfigToRuntime(selection.previousConfig); + } return; } + // Re-check: the confirm above awaits a GET, so a pick in that window would start a rival + // load over the same refs. Nothing awaits before the reservation below. + if (bailIfLoadInFlight()) return; + const forceCancelActive = stopDecision.forceCancelActive; const explicitIsLora = typeof selection === "string" ? undefined : selection.isLora; @@ -765,6 +827,10 @@ export function useChatModelRuntime() { upgrade: validation.transformers_upgrade, // No installable release: custom-code models may fall back to the trust_remote_code gate below. trustRemoteCodeFallback: validation.requires_trust_remote_code, + // The install refuses while chats generate and takes no force flag of its own, so + // without this the "Stop and reload" the user just confirmed dies here: Retry hits + // the same 409, and this path leaves chats running. + forceCancelActive, }); // The install unloads the previous model before the swap (even when // the swap then fails), so any exit after this point must roll back. @@ -808,7 +874,14 @@ export function useChatModelRuntime() { : undefined; if (currentCheckpoint) { - await unloadModel({ model_path: currentCheckpoint }); + // With chats generating, skip this preliminary unload: it cancels them ahead of /load's + // preflight, so a rejected target truncates replies for a model that never loads + // (/load evicts past those checks itself). Idle, unload first and free VRAM early. + if (!forceCancelActive) { + await unloadModel({ model_path: currentCheckpoint }); + } + // Set either way: /load can still leave no model resident, and an unneeded rollback + // hits already_loaded before the gate. previousWasUnloaded = true; } if (abortCtrl.signal.aborted) throw new Error("Cancelled"); @@ -915,6 +988,7 @@ export function useChatModelRuntime() { n_cpu_moe: loadNCpuMoe, tensor_split: loadSplitRatio ?? undefined, gpu_ids: loadSelectedGpuIds ?? undefined, + force_cancel_active: forceCancelActive, }); // If cancelled while loading, don't update UI to show @@ -1144,6 +1218,8 @@ export function useChatModelRuntime() { n_cpu_moe: stateBeforeUnload.loadedNCpuMoe ?? 0, tensor_split: stateBeforeUnload.loadedSplitRatio ?? undefined, gpu_ids: stateBeforeUnload.loadedGpuIds ?? undefined, + // The failed swap already unloaded the server those runs used. + force_cancel_active: true, }); const rollbackSpeculativeType = normalizeSpeculativeType( rollbackResponse.speculative_type, @@ -1540,13 +1616,15 @@ export function useChatModelRuntime() { if (!params.checkpoint) { return false; } - const runtime = useChatRuntimeStore.getState(); - if (runtime.modelLoading || runtime.loadingModelPick) { + const bailIfLoading = (): boolean => { + const runtime = useChatRuntimeStore.getState(); + if (!runtime.modelLoading && !runtime.loadingModelPick) return false; toast.info("A model is loading", { description: "Wait for it to finish or cancel it first.", }); - return false; - } + return true; + }; + if (bailIfLoading()) return false; setModelsError(null); if (isExternalModelId(params.checkpoint)) { clearCheckpoint(); @@ -1554,8 +1632,21 @@ export function useChatModelRuntime() { return true; } try { + // Ejecting tears down llama-server, so every chat stops. Same prompt, but it + // leaves no model loaded, so it must not be worded as a reload. + const stopDecision = await confirmStopRunningChatsIfNeeded( + "Unloading the model", + "unload", + ); + if (!stopDecision.proceed) return false; + // Same window as selectModel: a load may have started during the confirm. + if (bailIfLoading()) return false; + async function performUnload(): Promise { - await unloadModel({ model_path: params.checkpoint }); + await unloadModel({ + model_path: params.checkpoint, + force_cancel_active: stopDecision.forceCancelActive, + }); clearCheckpoint(); await refresh(); } diff --git a/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts b/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts index a08bd5fa54..651833102a 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts @@ -17,6 +17,7 @@ import { updateStoredChatThread, } from "../utils/chat-history-storage"; import { clearComposerDraft } from "../utils/composer-draft"; +import { stopChatThread } from "../utils/stop-chat-thread"; import { markChatThreadsDeleted, removeChatThreadTombstones, @@ -25,6 +26,8 @@ import { export interface SidebarItem { type: "single" | "compare"; id: string; + /** The pane threads behind this row id; `runningByThreadId` is keyed per pane thread. */ + threadIds?: string[]; title: string; createdAt: number; updatedAt: number; @@ -56,11 +59,13 @@ export function groupThreads( const existing = pairItems.get(t.pairId); if (existing) { existing.updatedAt = Math.max(existing.updatedAt, lastActivityAt(t)); + existing.threadIds?.push(t.id); continue; } const item: SidebarItem = { type: "compare", id: t.pairId, + threadIds: [t.id], title: t.title, createdAt: t.createdAt, updatedAt: lastActivityAt(t), @@ -160,10 +165,9 @@ export function useChatSidebarItems(options?: { } function cancelIfRunning(threadId: string): void { - const { runningByThreadId, cancelByThreadId } = - useChatRuntimeStore.getState(); - if (!runningByThreadId[threadId]) return; - cancelByThreadId[threadId]?.(); + // Reaches a background thread, which cancelByThreadId cannot: a deleted chat must stop, + // or the run keeps writing to a conversation that is gone. + stopChatThread(threadId); } export async function renameChatItem( diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts index 2c1bbcefad..a96a7509bd 100644 --- a/studio/frontend/src/features/chat/index.ts +++ b/studio/frontend/src/features/chat/index.ts @@ -55,8 +55,12 @@ export { export { preferFullToolOutput, toolOutputKey, + toolThreadScope, + useToolOutputFor, + useUnresolvedToolPaneScope, useToolPaneScope, } from "./tool-output-scope"; +export { useToolAwaitingApproval } from "./tool-approval"; export { PermissionModeDropdown } from "./permission-mode-select"; export { useChatSearchStore } from "./stores/chat-search-store"; export { usePinnedChatsStore } from "./stores/pinned-chats-store"; @@ -80,6 +84,7 @@ export { export { ApiProviderLogo } from "./api-provider-logo"; export { useExternalProvidersStore } from "./stores/external-providers-store"; export { ChatSearchDialog } from "./components/chat-search-dialog"; +export { StopRunningChatsDialog } from "./components/stop-running-chats-dialog"; export { setTrainingCompareHandoff } from "./lib/training-compare-handoff"; export type { ProjectRecord } from "./types"; export { clearAllChats, countAllChats } from "./utils/clear-all-chats"; diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index 2fa128bf2f..c22c753545 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -85,7 +85,11 @@ import { requestPromptQueueStop } from "./utils/prompt-queue-boundary"; import { isAssistantLocalThreadId } from "./utils/thread-ids"; const pendingHistoryAppendByMessageId = new Map>(); -const pendingRunStartReadyByMessageId = new Map>(); +// Resolves to the thread id assigned when this message's chat was first persisted. +const pendingRunStartReadyByMessageId = new Map< + string, + Promise +>(); type TitleResponse = { choices?: Array<{ @@ -699,6 +703,10 @@ function createStudioDbAdapter( async initialize(threadId: string) { await ensureThreadRecord({ threadId, modelType, pairId, projectId }); + // A run already streaming on this thread filed its handles under "__default" because + // the id did not exist yet. Re-key them now, or the sidebar row and Stop look up an + // id nothing is registered against. + useChatRuntimeStore.getState().adoptDefaultThreadRun(threadId); return { remoteId: threadId, externalId: undefined }; }, @@ -835,8 +843,8 @@ function trackHistoryAppend( function trackRunStartReady( messageId: string, - ready: Promise, -): Promise { + ready: Promise, +): Promise { pendingRunStartReadyByMessageId.set(messageId, ready); const cleanup = () => { setTimeout(() => { @@ -851,7 +859,7 @@ function trackRunStartReady( async function waitForRunStartHistoryAppend( messages: Parameters[0]["messages"], -): Promise { +): Promise { // Deep Research reserves an assistant placeholder before invoking the model // adapter, so the user message is not necessarily the final entry here. const userMessage = [...messages] @@ -862,15 +870,16 @@ async function waitForRunStartHistoryAppend( } const runStartReady = pendingRunStartReadyByMessageId.get(userMessage.id); const historyAppendReady = pendingHistoryAppendByMessageId.get(userMessage.id); - const pending = [runStartReady, historyAppendReady].filter( - (ready): ready is Promise => ready !== undefined, - ); - if (pending.length === 0) { - return; + if (runStartReady === undefined && historyAppendReady === undefined) { + return undefined; } let didBecomeReady = false; + let adoptedThreadId: string | undefined; try { - await Promise.all(pending); + [adoptedThreadId] = await Promise.all([ + runStartReady ?? Promise.resolve(undefined), + historyAppendReady?.then(() => undefined), + ]); didBecomeReady = true; } finally { if ( @@ -881,14 +890,22 @@ async function waitForRunStartHistoryAppend( pendingRunStartReadyByMessageId.delete(userMessage.id); } } + return adoptedThreadId; } function createPersistedRunAdapter(adapter: ChatModelAdapter): ChatModelAdapter { return { ...adapter, async *run(options) { - await waitForRunStartHistoryAppend(options.messages); - const result = adapter.run(options); + const adoptedThreadId = await waitForRunStartHistoryAppend(options.messages); + // The thread has an id by the time that resolves, but assistant-ui bound unstable_threadId + // before the await. Hand the run its real id so a first turn never files its handles + // under the unresolved key that concurrent runs share. + const result = adapter.run( + !options.unstable_threadId && adoptedThreadId + ? { ...options, unstable_threadId: adoptedThreadId } + : options, + ); if (!result) { return; } @@ -1153,7 +1170,13 @@ function useStudioRuntimeAdapters( : typeof store.ggufContextLength === "number" && store.ggufContextLength > 0; if (savedUsage && withinLocalLimit && modelMatches) { - store.setContextUsage(savedUsage); + // Key by the thread this loader read, not whichever is active when the await resolves: + // a switch inside it would file this thread's usage under the incoming one. Same rule + // the adapter's end-of-run write follows. + store.setThreadContextUsage(remoteId, savedUsage); + if (store.activeThreadId === remoteId) { + store.setContextUsage(savedUsage); + } } // If any message has a stored parentId, reconstruct the tree so @@ -1179,7 +1202,10 @@ function useStudioRuntimeAdapters( append({ parentId, message }: ExportedMessageRepositoryItem) { const initializeThread = aui.threadListItem().initialize(); - trackRunStartReady(message.id, initializeThread.then(() => undefined)); + trackRunStartReady( + message.id, + initializeThread.then(({ remoteId }) => remoteId), + ); const write = (async () => { const { remoteId } = await initializeThread; if (isChatThreadDeleted(remoteId)) { @@ -1308,17 +1334,6 @@ function createRuntimeHook(modelType: ModelType, pairId?: string) { }; } -function stopChatRun(threadId: string | null | undefined) { - if (!threadId) { - return; - } - try { - useChatRuntimeStore.getState().cancelByThreadId[threadId]?.(); - } catch { - // The run may have ended while navigation was mounting. - } -} - function ThreadAutoSwitch({ threadId, syncActiveThreadId = true, @@ -1333,8 +1348,9 @@ function ThreadAutoSwitch({ useEffect(() => { if (!isLoading && mainThreadId !== threadId) { if (syncActiveThreadId) { - requestPromptQueueStop(); - stopChatRun(mainThreadId); + // Stop queueing prompts to the outgoing thread but leave its run alone: its runtime + // stays mounted and keeps streaming. Only an explicit Stop cancels one. + requestPromptQueueStop({ cancelActiveRun: false }); } const switchResult = aui.threads().switchToThread(threadId) as unknown; if ( @@ -1365,16 +1381,14 @@ function ThreadNewChatSwitch({ }: { nonce: string }): ReactElement | null { const aui = useAui(); const isLoading = useAuiState(({ threads }) => threads.isLoading); - const mainThreadId = useAuiState(({ threads }) => threads.mainThreadId); - const mainThreadIdRef = useRef(mainThreadId); - mainThreadIdRef.current = mainThreadId; - + // The outgoing thread is not read here: New Chat leaves it running. useEffect(() => { if (isLoading) { return; } - requestPromptQueueStop(); - stopChatRun(mainThreadIdRef.current); + // New Chat leaves the previous conversation generating: its runtime stays mounted and + // the sidebar spins. Stopping it is its own Stop button's job. + requestPromptQueueStop({ cancelActiveRun: false }); // Switch to a fresh local thread without persisting it yet; persistence // still happens on first message append. void aui.threads().switchToNewThread(); diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 237cd857f0..98b8676c10 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -762,6 +762,30 @@ export function isDownloadableHubRepo(x: { ); } +type ContextUsageSnapshot = { + promptTokens: number; + completionTokens: number; + totalTokens: number; + cachedTokens: number; + // Anthropic-only; optional so pre-cache-stats persisted entries load. + cacheWriteTokens?: number; +}; + +/** + * One live run behind `runningByThreadId[id]`, with the `local` flag it started with so the + * model-swap gate can tell llama-server runs from external ones when runs share a key. + */ +type ThreadRunOwner = { + owner: () => void; + local: boolean; +}; + +type ToolStatusEntry = { + status: string; + startedAt: number; + owner?: () => void; +}; + type ChatRuntimeStore = { settingsHydrated: boolean; params: InferenceParams; @@ -771,7 +795,25 @@ type ChatRuntimeStore = { models: ChatModelSummary[]; loras: ChatLoraSummary[]; runningByThreadId: Record; + /** + * The subset of `runningByThreadId` decoding on the local llama-server. Swapping the local + * model neither interrupts an external-provider chat nor needs its consent, which is why + * the backend keeps those out of `active_generations` too. + */ + localRunByThreadId: Record; + /** + * Which runs set `runningByThreadId[id]`; see `setThreadRunning`'s `owner`. A list, not one + * entry: runs without a resolved thread id share the "__default" key, so one entry would let + * a newer run's clear delete an older run's flag while it still generates. + */ + runOwnerByThreadId: Record; cancelByThreadId: Record void>; + /** + * Backend cancels for the threads generating in the background. `cancelByThreadId` only holds + * the visible thread's `cancelRun()`, so the adapter parks a closure here that POSTs that + * run's own cancel_id. A list for the same reason as `runOwnerByThreadId`: "__default" is shared. + */ + serverCancelByThreadId: Record void)[]>; autoTitle: boolean; hfToken: string; modelsError: string | null; @@ -892,7 +934,16 @@ type ChatRuntimeStore = { * consulted when `providerSupportsBuiltinWebFetch` is true. */ webFetchToolsEnabled: boolean; - toolStatus: string | null; + /** + * Live tool status per conversation ("Running Python: ...") with its start time. Keyed by + * thread, or one chat's tool call shows above every other composer; the timestamp keeps the + * counter running across a thread switch. + */ + /** + * Per-run entries, newest last. Unresolved threads share "__default", so one scalar per key + * meant a finishing run's clear removed a sibling's status while its tool was still running. + */ + toolStatusByThreadId: Record; /** Live stdout/stderr from running tools, keyed by toolCallId. Transient: * appended by tool_output, cleared on tool_end or run end. */ toolLiveOutput: Record; @@ -959,9 +1010,12 @@ type ChatRuntimeStore = { /** Active model is a block-diffusion model (DiffusionGemma): drives the * denoising-canvas artifact auto-render. */ loadedIsDiffusion: boolean; - /** Live denoising frame for the in-progress diffusion message. Transient: set - * per step, cleared when the run ends, never persisted into the transcript. */ - activeDiffusionCanvas: DiffusionCanvasFrame | null; + /** + * Live denoising frame per conversation ("__default" until the id exists). Transient: set per + * step, cleared when the run ends, never persisted. Keyed, not global: two denoising chats + * overwrote each other's frame, so the visible preview flickered or vanished. + */ + activeDiffusionCanvasByThreadId: Record; customContextLength: number | null; /** The pinned context the loaded model used (null = Auto), so dirty-tracking * and a later fit Apply can tell an explicit pin apart from Auto. */ @@ -984,14 +1038,13 @@ type ChatRuntimeStore = { pendingAudioBase64: string | null; pendingAudioName: string | null; pendingImageEditReference: PendingImageEditReference | null; - contextUsage: { - promptTokens: number; - completionTokens: number; - totalTokens: number; - cachedTokens: number; - // Anthropic-only; optional so pre-cache-stats persisted entries load. - cacheWriteTokens?: number; - } | null; + contextUsage: ContextUsageSnapshot | null; + /** + * Per-thread copy of the above, so the bar survives a switch away and back. `contextUsage` is + * the VISIBLE conversation's usage and a background run may not write it, so without this a + * run finishing off-screen leaves nothing to restore. + */ + contextUsageByThreadId: Record; modelLoading: boolean; loadingModelPick: LoadingModelPick | null; activeNativePathToken: string | null; @@ -1010,9 +1063,35 @@ type ChatRuntimeStore = { setActivePresetSource: (source: ChatPresetSource) => void; setModels: (models: ChatModelSummary[]) => void; setLoras: (loras: ChatLoraSummary[]) => void; - setThreadRunning: (threadId: string, running: boolean) => void; + /** + * `local` defaults to true, so an unqualified caller still counts for the model-swap gate. + * `owner` narrows the clear to the run that set the flag: unresolved thread ids share the + * "__default" key, so a blind delete would drop a sibling's live entry. Owners accumulate, + * so the flag survives until the last one clears. + */ + setThreadRunning: ( + threadId: string, + running: boolean, + options?: { local?: boolean; owner?: () => void }, + ) => void; + /** + * Re-key a first turn's run handles once its thread is persisted. + * + * A run that starts before its id exists files everything under "__default". Nothing moved it + * afterwards, so once the user navigated away the sidebar found no run and showed no spinner; + * stopChatThread had no handle either and the generation carried on holding a slot. + */ + adoptDefaultThreadRun: (threadId: string) => void; + /** + * Which key this run's handles live under now. `adoptDefaultThreadRun` re-keys them mid-run, + * so a run that started under "__default" must look its owner up instead of reusing the key + * it captured, or its writes and its final clear miss the entries. + */ + runKeyForOwner: (fallbackKey: string, owner: () => void) => string; registerThreadCancel: (threadId: string, cancel: () => void) => void; clearThreadCancel: (threadId: string) => void; + registerThreadServerCancel: (threadId: string, cancel: () => void) => void; + clearThreadServerCancel: (threadId: string, cancel?: () => void) => void; setAutoTitle: (enabled: boolean) => void; setHfToken: (token: string) => void; setModelsError: (error: string | null) => void; @@ -1066,7 +1145,15 @@ type ChatRuntimeStore = { setRagAutoInjectMinScore: (score: number) => void; setRagOcrScanned: (enabled: boolean) => void; setRagCaptionFigures: (enabled: boolean) => void; - setToolStatus: (status: string | null) => void; + /** + * `owner` is the run's identity token, as for `setThreadRunning`: unresolved threads share + * "__default", so without it one run's cleanup clears a concurrent run's status. + */ + setToolStatus: ( + threadId: string, + status: string | null, + owner?: () => void, + ) => void; appendToolLiveOutput: (toolCallId: string, text: string) => void; /** Clear one tool's live output, or all when no id is given. */ clearToolLiveOutput: (toolCallId?: string) => void; @@ -1075,7 +1162,13 @@ type ChatRuntimeStore = { /** Drop a stale preserved full output (a new run is reusing the id). */ clearToolFullOutput: (toolCallId: string) => void; setGeneratingStatus: (status: string | null) => void; - setActiveDiffusionCanvas: (canvas: DiffusionCanvasFrame | null) => void; + setActiveDiffusionCanvas: ( + threadId: string | null, + canvas: DiffusionCanvasFrame, + ) => void; + /** Drop only `threadId`'s canvas: a run ending in a background chat must not wipe the + * frame another chat is still painting. */ + clearActiveDiffusionCanvasForThread: (threadId: string | null) => void; setAutoHealToolCalls: (enabled: boolean) => void; setNudgeToolCalls: (enabled: boolean) => void; setMaxToolCallsPerMessage: (value: number) => void; @@ -1095,6 +1188,11 @@ type ChatRuntimeStore = { ) => void; clearPendingImageEditReference: () => void; setContextUsage: (usage: ChatRuntimeStore["contextUsage"]) => void; + /** A finished run's usage, kept per thread so switching back re-applies it. */ + setThreadContextUsage: ( + threadId: string, + usage: ContextUsageSnapshot, + ) => void; }; type PersistedChatSettings = Awaited< @@ -1310,7 +1408,10 @@ export const useChatRuntimeStore = create((set, get) => ({ models: [], loras: [], runningByThreadId: {}, + localRunByThreadId: {}, + runOwnerByThreadId: {}, cancelByThreadId: {}, + serverCancelByThreadId: {}, autoTitle: false, hfToken: useHfTokenStore.getState().token, modelsError: null, @@ -1374,11 +1475,11 @@ export const useChatRuntimeStore = create((set, get) => ({ ), ragOcrScanned: loadBool(CHAT_RAG_OCR_KEY, DEFAULT_RAG_OCR), ragCaptionFigures: loadBool(CHAT_RAG_CAPTION_KEY, DEFAULT_RAG_CAPTION), - toolStatus: null, + toolStatusByThreadId: {}, toolLiveOutput: {}, toolFullOutput: {}, generatingStatus: null, - activeDiffusionCanvas: null, + activeDiffusionCanvasByThreadId: {}, autoHealToolCalls: true, nudgeToolCalls: true, maxToolCallsPerMessage: 25, @@ -1423,6 +1524,7 @@ export const useChatRuntimeStore = create((set, get) => ({ pendingAudioName: null, pendingImageEditReference: null, contextUsage: null, + contextUsageByThreadId: {}, modelLoading: false, loadingModelPick: null, activeNativePathToken: null, @@ -1495,7 +1597,9 @@ export const useChatRuntimeStore = create((set, get) => ({ const checkpointChanged = state.params.checkpoint !== params.checkpoint; return { params, - ...(checkpointChanged ? { contextUsage: null } : {}), + ...(checkpointChanged + ? { contextUsage: null, contextUsageByThreadId: {} } + : {}), }; }), setCustomPresets: (customPresets) => @@ -1518,16 +1622,94 @@ export const useChatRuntimeStore = create((set, get) => ({ }), setModels: (models) => set({ models }), setLoras: (loras) => set({ loras }), - setThreadRunning: (threadId, running) => + setThreadRunning: (threadId, running, options) => set((state) => { const next = { ...state.runningByThreadId }; + const nextLocal = { ...state.localRunByThreadId }; + const nextOwner = { ...state.runOwnerByThreadId }; + const owners = state.runOwnerByThreadId[threadId] ?? []; + const local = options?.local !== false; if (running) { next[threadId] = true; + if (options?.owner) { + nextOwner[threadId] = [...owners, { owner: options.owner, local }]; + } + // Any local owner keeps the key counted by the model-swap gate, so an external run + // joining a shared key must not clear a sibling's flag. + if (local) { + nextLocal[threadId] = true; + } else if (!owners.some((o) => o.local)) { + delete nextLocal[threadId]; + } } else { - delete next[threadId]; + const remaining = options?.owner + ? owners.filter((o) => o.owner !== options.owner) + : []; + // An owner missing from the list was already cleared, or the key belongs to siblings + // only: either way this run must change nothing. + if (options?.owner && remaining.length === owners.length) return state; + // An ownerless clear predates per-run tracking, so it must not speak for runs that + // own the key: leave them to clear themselves. + if (!options?.owner && owners.length > 0) return state; + if (remaining.length > 0) { + nextOwner[threadId] = remaining; + if (remaining.some((o) => o.local)) { + nextLocal[threadId] = true; + } else { + delete nextLocal[threadId]; + } + } else { + delete next[threadId]; + delete nextLocal[threadId]; + delete nextOwner[threadId]; + } } - return { runningByThreadId: next }; + return { + runningByThreadId: next, + localRunByThreadId: nextLocal, + runOwnerByThreadId: nextOwner, + }; }), + adoptDefaultThreadRun: (threadId) => + set((state) => { + const key = "__default"; + if (!threadId || threadId === key) return state; + // Two first turns can share "__default", and nothing links a run there to the thread being + // persisted. Moving the arrays wholesale handed this thread the sibling's owner and stop + // handle too, so stopping one aborted both. Adopt only when the key holds a single run. + if ((state.runOwnerByThreadId[key]?.length ?? 0) > 1) return state; + // Only the transient run maps move. Anything already filed under the real id wins, + // since that is a later, better-identified run. + const moved: Partial = {}; + const move = ( + map: Record, + name: keyof ChatRuntimeStore, + ) => { + const entry = map[key]; + if (entry === undefined || map[threadId] !== undefined) return; + const next = { ...map }; + delete next[key]; + next[threadId] = entry; + (moved as Record)[name as string] = next; + }; + move(state.runningByThreadId, "runningByThreadId"); + move(state.localRunByThreadId, "localRunByThreadId"); + move(state.runOwnerByThreadId, "runOwnerByThreadId"); + move(state.cancelByThreadId, "cancelByThreadId"); + move(state.serverCancelByThreadId, "serverCancelByThreadId"); + move(state.toolStatusByThreadId, "toolStatusByThreadId"); + move( + state.activeDiffusionCanvasByThreadId, + "activeDiffusionCanvasByThreadId", + ); + return Object.keys(moved).length > 0 ? moved : state; + }), + runKeyForOwner: (fallbackKey, owner) => { + for (const [key, entries] of Object.entries(get().runOwnerByThreadId)) { + if (entries.some((e) => e.owner === owner)) return key; + } + return fallbackKey; + }, registerThreadCancel: (threadId, cancel) => set((state) => { const next = { ...state.cancelByThreadId }; @@ -1541,6 +1723,29 @@ export const useChatRuntimeStore = create((set, get) => ({ delete next[threadId]; return { cancelByThreadId: next }; }), + registerThreadServerCancel: (threadId, cancel) => + set((state) => { + const next = { ...state.serverCancelByThreadId }; + next[threadId] = [...(state.serverCancelByThreadId[threadId] ?? []), cancel]; + return { serverCancelByThreadId: next }; + }), + // `cancel` narrows removal to the run that registered it: unresolved thread ids share the + // "__default" key, so a blind delete would drop a live sibling. + clearThreadServerCancel: (threadId, cancel) => + set((state) => { + const current = state.serverCancelByThreadId[threadId]; + if (current === undefined) return state; + const remaining = + cancel === undefined ? [] : current.filter((c) => c !== cancel); + if (remaining.length === current.length) return state; + const next = { ...state.serverCancelByThreadId }; + if (remaining.length > 0) { + next[threadId] = remaining; + } else { + delete next[threadId]; + } + return { serverCancelByThreadId: next }; + }), setAutoTitle: (autoTitle) => set((state) => { setScalarSettingVersion("autoTitle", autoTitle, state.autoTitle); @@ -1588,14 +1793,24 @@ export const useChatRuntimeStore = create((set, get) => ({ maxTokens: nextMaxTokens, }, activeGgufVariant: ggufVariant ?? null, - ...(checkpointChanged ? { contextUsage: null } : {}), + ...(checkpointChanged + ? { contextUsage: null, contextUsageByThreadId: {} } + : {}), // Switching to an external provider disables Deep Research, which only // applies to the local base model. ...(isExternalModelId(modelId) ? { deepResearchEnabled: false } : {}), }; }), + // Re-apply the incoming thread's own usage rather than blanking the bar: a run that finished + // in the background never wrote the visible value, and a still-mounted runtime skips the + // history loader on the way back. setActiveThreadId: (activeThreadId) => - set({ activeThreadId, contextUsage: null }), + set((state) => ({ + activeThreadId, + contextUsage: activeThreadId + ? (state.contextUsageByThreadId[activeThreadId] ?? null) + : null, + })), setActiveProjectId: (activeProjectId) => set({ activeProjectId }), setIncognito: (incognito) => { if (incognito) saveBool(CHAT_DEEP_RESEARCH_ENABLED_KEY, false); @@ -1626,6 +1841,7 @@ export const useChatRuntimeStore = create((set, get) => ({ ggufNativeContextLength: null, modelRequiresTrustRemoteCode: false, contextUsage: null, + contextUsageByThreadId: {}, supportsReasoning: false, reasoningAlwaysOn: false, reasoningEnabled: true, @@ -1647,10 +1863,10 @@ export const useChatRuntimeStore = create((set, get) => ({ webFetchToolsEnabled: false, // Only the per-session enable pill resets; source/mode/top_k persist. ragEnabled: false, - toolStatus: null, + toolStatusByThreadId: {}, toolLiveOutput: {}, toolFullOutput: {}, - activeDiffusionCanvas: null, + activeDiffusionCanvasByThreadId: {}, kvCacheDtype: null, loadedKvCacheDtype: null, speculativeType: readPersistedSpeculativeType(), @@ -1945,7 +2161,31 @@ export const useChatRuntimeStore = create((set, get) => ({ saveBool(CHAT_RAG_CAPTION_KEY, ragCaptionFigures); return { ragCaptionFigures }; }), - setToolStatus: (toolStatus) => set({ toolStatus }), + setToolStatus: (threadId, status, owner) => + set((state) => { + const next = { ...state.toolStatusByThreadId }; + const entries = state.toolStatusByThreadId[threadId] ?? []; + const mine = entries.find((e) => e.owner === owner); + if (!status) { + // Drop only this run's entry: a sibling behind the same key may still be running a tool, + // and its status has to survive this clear. + if (mine === undefined) return state; + const rest = entries.filter((e) => e !== mine); + if (rest.length > 0) { + next[threadId] = rest; + } else { + delete next[threadId]; + } + } else { + // Same text from the same run means the same call, so keep startedAt: only a new tool restarts it. + if (mine?.status === status) return state; + const entry = { status, startedAt: Date.now(), owner }; + next[threadId] = mine + ? entries.map((e) => (e === mine ? entry : e)) + : [...entries, entry]; + } + return { toolStatusByThreadId: next }; + }), appendToolLiveOutput: (toolCallId, text) => set((state) => ({ toolLiveOutput: { @@ -1983,8 +2223,21 @@ export const useChatRuntimeStore = create((set, get) => ({ delete next[toolCallId]; return { toolLiveOutput: next }; }), - setActiveDiffusionCanvas: (activeDiffusionCanvas) => - set({ activeDiffusionCanvas }), + setActiveDiffusionCanvas: (threadId, canvas) => + set((state) => ({ + activeDiffusionCanvasByThreadId: { + ...state.activeDiffusionCanvasByThreadId, + [threadId || "__default"]: canvas, + }, + })), + clearActiveDiffusionCanvasForThread: (threadId) => + set((state) => { + const key = threadId || "__default"; + if (state.activeDiffusionCanvasByThreadId[key] === undefined) return state; + const next = { ...state.activeDiffusionCanvasByThreadId }; + delete next[key]; + return { activeDiffusionCanvasByThreadId: next }; + }), setGeneratingStatus: (generatingStatus) => set({ generatingStatus }), setAutoHealToolCalls: (autoHealToolCalls) => set((state) => { @@ -2050,7 +2303,27 @@ export const useChatRuntimeStore = create((set, get) => ({ set({ pendingImageEditReference }), clearPendingImageEditReference: () => set({ pendingImageEditReference: null }), - setContextUsage: (contextUsage) => set({ contextUsage }), + // Write through to the visible thread's own entry, so a value restored by the history loader + // survives a switch away and back: that loader runs once per mount and setActiveThreadId + // reads the map, so without this the bar goes blank on return. + setContextUsage: (contextUsage) => + set((state) => { + if (!state.activeThreadId) return { contextUsage }; + const next = { ...state.contextUsageByThreadId }; + if (contextUsage) { + next[state.activeThreadId] = contextUsage; + } else { + delete next[state.activeThreadId]; + } + return { contextUsage, contextUsageByThreadId: next }; + }), + setThreadContextUsage: (threadId, usage) => + set((state) => ({ + contextUsageByThreadId: { + ...state.contextUsageByThreadId, + [threadId]: usage, + }, + })), })); // Mirror token edits made through the shared store (e.g. Unsloth's field). diff --git a/studio/frontend/src/features/chat/stores/stop-running-chats-dialog-store.ts b/studio/frontend/src/features/chat/stores/stop-running-chats-dialog-store.ts new file mode 100644 index 0000000000..01ccc76f43 --- /dev/null +++ b/studio/frontend/src/features/chat/stores/stop-running-chats-dialog-store.ts @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; + +type Resolver = (confirmed: boolean) => void; + +/** What confirming does to the model: reload it, or leave none loaded. */ +export type StopRunningChatsEffect = "reload" | "unload"; + +// One at a time: a new request declines any pending one so no promise leaks. +let pendingResolver: Resolver | null = null; + +interface StopRunningChatsDialogStore { + open: boolean; + /** How many conversations the pending action would stop. */ + count: number; + /** Titles of those conversations, when known, for the dialog body. */ + titles: string[]; + /** What the user is about to do, e.g. "Loading a different model". */ + action: string; + /** The set includes an embeddings/completions/audio request, which is not a chat. */ + hasNonChat: boolean; + /** Ejecting leaves no model loaded, so it must not be described as a reload. */ + effect: StopRunningChatsEffect; + requestConfirm: (args: { + count: number; + titles?: string[]; + action?: string; + hasNonChat?: boolean; + effect?: StopRunningChatsEffect; + }) => Promise; + resolve: (confirmed: boolean) => void; +} + +export const useStopRunningChatsDialogStore = + create()((set) => ({ + open: false, + count: 0, + titles: [], + action: "", + hasNonChat: false, + effect: "reload", + requestConfirm: ({ + count, + titles = [], + action = "", + hasNonChat = false, + effect = "reload", + }) => + new Promise((resolve) => { + pendingResolver?.(false); + pendingResolver = resolve; + set({ open: true, count, titles, action, hasNonChat, effect }); + }), + resolve: (confirmed) => { + const resolver = pendingResolver; + pendingResolver = null; + set({ + open: false, + count: 0, + titles: [], + action: "", + hasNonChat: false, + effect: "reload", + }); + resolver?.(confirmed); + }, + })); diff --git a/studio/frontend/src/features/chat/tool-approval.ts b/studio/frontend/src/features/chat/tool-approval.ts new file mode 100644 index 0000000000..b1908dacc0 --- /dev/null +++ b/studio/frontend/src/features/chat/tool-approval.ts @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"use client"; + +import { useChatRuntimeStore } from "./stores/chat-runtime-store"; + +/** + * True while this card's call is parked on the Allow / Deny prompt, so it can say it is + * waiting rather than counting up "Running". Set when the backend gates the call. + */ +export function useToolAwaitingApproval(toolCallId?: string): boolean { + return useChatRuntimeStore( + (s) => + !!toolCallId && + Object.prototype.hasOwnProperty.call(s.toolConfirmations, toolCallId), + ); +} diff --git a/studio/frontend/src/features/chat/tool-output-scope.ts b/studio/frontend/src/features/chat/tool-output-scope.ts index a7885a3916..abf8163431 100644 --- a/studio/frontend/src/features/chat/tool-output-scope.ts +++ b/studio/frontend/src/features/chat/tool-output-scope.ts @@ -3,6 +3,7 @@ "use client"; +import { useAuiState } from "@assistant-ui/react"; import { createContext, useContext } from "react"; import type { ModelType } from "./types"; @@ -20,10 +21,58 @@ export function toolPaneScope(modelType?: ModelType, pairId?: string): string { return `${modelType ?? "base"}\u0000${pairId ?? ""}`; } +/** + * Narrow a pane scope to one conversation: two threads in a pane can both be mid "call_0", + * so without the thread in the key they share a store entry and swap outputs. + */ +export function toolThreadScope(paneScope: string, threadId?: string): string { + return `${paneScope}\u0000${threadId ?? ""}`; +} + export const ToolPaneScopeContext = createContext(toolPaneScope()); +/** + * Store-key scope for the conversation this component renders in, taken from the surrounding + * runtime so reader and writer agree without a prop. + * + * `remoteId`, not `id`: the adapter gets `unstable_threadId`, which assistant-ui sources from + * `remoteId`, and an uninitialized thread has `id` but no `remoteId`. Reading `id` split the + * keys apart for the first turn of every New Chat, so live tool output never reached the card. + */ export function useToolPaneScope(): string { - return useContext(ToolPaneScopeContext); + const paneScope = useContext(ToolPaneScopeContext); + const threadId = useAuiState(({ threadListItem }) => threadListItem.remoteId); + return toolThreadScope(paneScope, threadId); +} + +/** + * Read a tool-output map for one call, tolerating a run that started before its thread had an id. + * + * The adapter captures its scope once at run start, so a first turn writes under the unresolved + * scope for its whole life. The autosave can assign `remoteId` mid-run, which moves this + * component's key but not the writer's, and the card went blank. Falling back to the pane-wide + * scope keeps those entries reachable; only an unpersisted first turn can be filed there. + */ +/** The scope a run that started before its thread had an id writes under. */ +export function useUnresolvedToolPaneScope(): string { + return toolThreadScope(useContext(ToolPaneScopeContext), undefined); +} + +export function useToolOutputFor( + map: Record, + paneScope: string, + toolCallId: string, +): string { + // Unconditional: hooks cannot sit behind the early return below. + const unresolvedScope = useUnresolvedToolPaneScope(); + // Only a thread mid-run can be the one that just gained its id. Local ids repeat + // ("call_0"), so an unconditional fallback showed a live first turn's stdout in every + // older conversation whose own entry had been cleared. + const isRunning = useAuiState(({ thread }) => thread.isRunning); + const own = map[toolOutputKey(paneScope, toolCallId)]; + if (own !== undefined) return own; + if (!isRunning) return ""; + return map[toolOutputKey(unresolvedScope, toolCallId)] ?? ""; } /** Store key for the live/full tool output maps: pane scope + tool call id. */ diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index d554eb777e..3681b0f0cb 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -35,6 +35,11 @@ export interface ListLorasResponse { export interface LoadModelRequest { model_path: string; + /** + * Stop any chats still generating instead of getting a 409: a load replaces the single + * llama-server they all decode on. Set only after the user confirms. + */ + force_cancel_active?: boolean; nativePathLease?: string | null; hf_token: string | null; max_seq_length: number; @@ -201,6 +206,9 @@ export interface LoadModelResponse { export interface UnloadModelRequest { model_path: string; + /** Stop any chats still generating instead of getting a 409: the unload takes down the + * llama-server they all decode on. */ + force_cancel_active?: boolean; } export interface InferenceStatusResponse { diff --git a/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts b/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts new file mode 100644 index 0000000000..7b20ceb362 --- /dev/null +++ b/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { getActiveGenerations } from "../api/chat-api"; +import { useChatRuntimeStore } from "../stores/chat-runtime-store"; +import { + type StopRunningChatsEffect, + useStopRunningChatsDialogStore, +} from "../stores/stop-running-chats-dialog-store"; +import { listStoredChatThreads } from "./chat-history-storage"; + +export interface StopRunningChatsDecision { + /** False when the user chose to keep generating; the caller must not load. */ + proceed: boolean; + /** Pass as `force_cancel_active`. True only after an explicit confirmation, so the backend's 409 still guards every other caller. */ + forceCancelActive: boolean; +} + +/** + * Gate a model load / reload on the chats still generating: they share one llama-server, + * so a reload ends all of them. Ask first, then let the backend cancel them once the load + * is past preflight. External-provider chats are left out of both. + */ +export async function confirmStopRunningChatsIfNeeded( + action = "Loading a different model", + effect: StopRunningChatsEffect = "reload", +): Promise { + // Local runs only: an external-provider chat is not stopped by the swap, so counting it + // would block a safe load behind a dialog. The backend excludes them for the same reason. + const { runningByThreadId, localRunByThreadId } = + useChatRuntimeStore.getState(); + let running = Object.entries(runningByThreadId) + .filter(([threadId, on]) => on && localRunByThreadId[threadId]) + .map(([threadId]) => threadId); + let count = running.length; + let hasNonChat = false; + + // Always merge the backend snapshot: runningByThreadId is this tab's memory, empty after a + // reload and blind to a second tab, while force_cancel_active cancels every backend run. + // The union stays local-only, since external-provider runs are never in it. + try { + const active = await getActiveGenerations(); + const entries = active.active ?? []; + const merged = new Set(running); + for (const threadId of active.thread_ids ?? []) { + merged.add(threadId); + } + running = [...merged]; + // Count conversations, not handles: one chat holds several at once while a tool + // continuation registers its next leg before the previous unwinds, and active.count + // counts those separately. A first turn started before its id was persisted has no + // id to merge, so add those back or the prompt names fewer chats than will stop. + const unnamed = entries.filter((entry) => !entry.thread_id).length; + count = entries.length + ? running.length + unnamed + : Math.max(active.count ?? 0, running.length); + // Embeddings / completions / audio share the model but are not conversations, so the + // prompt must not offer to stop chats that do not exist. + hasNonChat = entries.some((entry) => (entry.kind ?? "chat") !== "chat"); + } catch { + // Backend unreachable / older build: fall back to the local map only. + } + + if (count === 0) { + return { proceed: true, forceCancelActive: false }; + } + + let titles: string[] = []; + try { + const threads = await listStoredChatThreads(); + const byId = new Map(threads.map((t) => [t.id, t])); + // A compare conversation runs two pane threads, and the sidebar and the route both treat + // it as one chat. Counting the raw ids asked to stop two and listed its title twice. Fold + // panes onto their pairId, keeping the backend's count when it is higher. + const seen = new Set(); + for (const id of running) { + const thread = byId.get(id); + const key = thread?.pairId ?? id; + if (seen.has(key)) continue; + seen.add(key); + titles.push(thread?.title || "Untitled chat"); + } + count = Math.max(seen.size, count - (running.length - seen.size)); + } catch { + // Titles are decoration; the count alone is enough to make the choice. + titles = []; + } + + const confirmed = await useStopRunningChatsDialogStore + .getState() + .requestConfirm({ count, titles, action, hasNonChat, effect }); + + if (!confirmed) { + return { proceed: false, forceCancelActive: false }; + } + + // Deliberately no local stop: the backend holds the cancel until the load clears preflight, + // so stopping now would truncate every chat even for a rejected load. + return { proceed: true, forceCancelActive: true }; +} diff --git a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts index ac1d973bfe..8013917ec8 100644 --- a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts +++ b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts @@ -1,8 +1,17 @@ export const PROMPT_QUEUE_STOP_EVENT = "unsloth:prompt-queue-stop"; -export function requestPromptQueueStop() { +export interface PromptQueueStopOptions { + /** Also cancel the prompt the queue already dispatched. Navigation passes `false` to + * leave it generating; an explicit stop passes `true` (the default). */ + cancelActiveRun?: boolean; +} + +export function requestPromptQueueStop(options: PromptQueueStopOptions = {}) { if (typeof window === "undefined") { return; } - window.dispatchEvent(new Event(PROMPT_QUEUE_STOP_EVENT)); + const { cancelActiveRun = true } = options; + window.dispatchEvent( + new CustomEvent(PROMPT_QUEUE_STOP_EVENT, { detail: { cancelActiveRun } }), + ); } diff --git a/studio/frontend/src/features/chat/utils/stop-chat-thread.ts b/studio/frontend/src/features/chat/utils/stop-chat-thread.ts new file mode 100644 index 0000000000..ba0fac9aa9 --- /dev/null +++ b/studio/frontend/src/features/chat/utils/stop-chat-thread.ts @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { useChatRuntimeStore } from "../stores/chat-runtime-store"; + +/** + * Stop one conversation's generation, visible or not. Returns true if a stop was dispatched. + * + * `cancelByThreadId` is assistant-ui's `cancelRun()`, registered only for the thread on screen; + * `serverCancelByThreadId` is registered for every run and POSTs that run's own `cancel_id`, so + * it is the only handle a background conversation has. Both are per-run. Runs with an unresolved + * thread id share the "__default" key, so stop every handle filed under it. + */ +export function stopChatThread(threadId: string | null | undefined): boolean { + if (!threadId) return false; + const { runningByThreadId, cancelByThreadId, serverCancelByThreadId } = + useChatRuntimeStore.getState(); + if (!runningByThreadId[threadId]) return false; + let stopped = false; + try { + const cancel = cancelByThreadId[threadId]; + if (cancel) { + cancel(); + stopped = true; + } + } catch { + // The run may have ended between the read above and this call. + } + // Also after cancelRun(): a proxy that swallows the fetch abort leaves the backend decoding. + for (const serverCancel of serverCancelByThreadId[threadId] ?? []) { + try { + serverCancel(); + stopped = true; + } catch { + // Same as above. + } + } + return stopped; +} diff --git a/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts b/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts index 2df9f712ff..7e1e1b0f5a 100644 --- a/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts +++ b/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts @@ -16,14 +16,19 @@ interface InstallLatestTransformersResponse { latest_version?: string | null; } -/** Consented install of the latest transformers into the sidecar; synchronous, can take minutes. */ +/** Consented install of the latest transformers into the sidecar; synchronous, can take minutes. + * + * `forceCancelActive` carries the answer the user already gave the model swap's "stop N + * chats" prompt: without it the install 409s while those chats run, and nothing between the + * two dialogs stops them. Only ever true after that confirmation. */ export async function installLatestTransformers( version: string, + forceCancelActive = false, ): Promise { const response = await authFetch("/api/inference/install-latest-transformers", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ version }), + body: JSON.stringify({ version, force_cancel_active: forceCancelActive }), }); if (!response.ok) { throw new Error(await readFastApiError(response)); diff --git a/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts b/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts index 7d79d08d9c..9b942a7008 100644 --- a/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts +++ b/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts @@ -10,6 +10,9 @@ interface ConfirmArgs { upgrade: TransformersUpgradeInfo | null | undefined; /** When no release is installable, offer continuing into the caller's custom-code gate. */ trustRemoteCodeFallback?: boolean; + /** The caller already confirmed the swap's "stop N chats" prompt: carry it into + * the install, which otherwise 409s on those same chats with no way forward. */ + forceCancelActive?: boolean; } /** Pause a load needing a newer transformers on the consent dialog and run the install. @@ -18,11 +21,13 @@ export async function confirmTransformersUpgradeIfNeeded({ modelName, upgrade, trustRemoteCodeFallback, + forceCancelActive, }: ConfirmArgs): Promise { if (!upgrade) return true; return useTransformersUpgradeDialogStore .getState() .requestConsent(modelName, upgrade, { trustRemoteCodeFallback: Boolean(trustRemoteCodeFallback), + forceCancelActive: Boolean(forceCancelActive), }); } diff --git a/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts b/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts index 9e307fb1a1..be37691bc6 100644 --- a/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts +++ b/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts @@ -18,6 +18,9 @@ interface TransformersUpgradeDialogStore { errorMessage: string | null; /** Model ships custom code; without a PyPI install the load may fall back to trust_remote_code. */ trustRemoteCodeFallback: boolean; + /** The caller already confirmed the model swap's "stop N chats" prompt, so the install + * may stop them too; without it the install 409s and Retry can never succeed. */ + forceCancelActive: boolean; /** True once this consent's install completed. The install unloads the previous * model before swapping, so the caller must treat it as already unloaded; the * custom-code fallback resolves true without installing and leaves it loaded. */ @@ -34,7 +37,7 @@ interface TransformersUpgradeDialogStore { requestConsent: ( modelName: string, upgrade: TransformersUpgradeInfo, - options?: { trustRemoteCodeFallback?: boolean }, + options?: { trustRemoteCodeFallback?: boolean; forceCancelActive?: boolean }, ) => Promise; /** Accept/Retry: run the install; on success resolve(true) and close. */ install: () => Promise; @@ -49,6 +52,7 @@ export const useTransformersUpgradeDialogStore = phase: "consent", errorMessage: null, trustRemoteCodeFallback: false, + forceCancelActive: false, installRan: false, serverUnloadedChat: false, requestConsent: (modelName, upgrade, options) => @@ -62,6 +66,7 @@ export const useTransformersUpgradeDialogStore = phase: "consent", errorMessage: null, trustRemoteCodeFallback: Boolean(options?.trustRemoteCodeFallback), + forceCancelActive: Boolean(options?.forceCancelActive), installRan: false, }); }), @@ -71,14 +76,14 @@ export const useTransformersUpgradeDialogStore = return value; }, install: async () => { - const { upgrade, phase } = get(); + const { upgrade, phase, forceCancelActive } = get(); const version = upgrade?.pypi_version; if (!version || phase === "installing") return; const requestResolver = pendingResolver; set({ phase: "installing", errorMessage: null }); let result: Awaited>; try { - result = await installLatestTransformers(version); + result = await installLatestTransformers(version, forceCancelActive); // Latch the server-side unload IMMEDIATELY, before any resolver-identity // guard: even a superseded consent's install may have unloaded the chat // model, and the signal must survive for whichever load consumes it next. @@ -133,6 +138,7 @@ export const useTransformersUpgradeDialogStore = phase: "consent", errorMessage: null, trustRemoteCodeFallback: false, + forceCancelActive: false, }); resolver?.(installed); }, diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts index 4b2a9e5ec0..47d5032fae 100644 --- a/studio/frontend/src/i18n/locales/ar.ts +++ b/studio/frontend/src/i18n/locales/ar.ts @@ -37,6 +37,8 @@ export const ar = { navigation: { newChat: "محادثة جديدة", returnToChat: "العودة إلى المحادثة", + returnToChats: "العودة إلى {count} محادثات", + chatGenerating: "جارٍ الإنشاء", compare: "مقارنة", search: "بحث", hub: "مركز النماذج", diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts index a508fbb9fc..cb7d603f42 100644 --- a/studio/frontend/src/i18n/locales/de.ts +++ b/studio/frontend/src/i18n/locales/de.ts @@ -37,6 +37,8 @@ export const de = { navigation: { newChat: "Neuer Chat", returnToChat: "Zurück zum Chat", + returnToChats: "Zurück zu {count} Chats", + chatGenerating: "Wird generiert", compare: "Vergleichen", search: "Suchen", hub: "Modell-Hub", diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index 955a876dd5..bdfcf38231 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -34,6 +34,8 @@ export const en = { navigation: { newChat: "New chat", returnToChat: "Return to Chat", + returnToChats: "Return to {count} Chats", + chatGenerating: "Generating", compare: "Compare", search: "Search", hub: "Model hub", diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts index 26e5e062dd..f7cb0e11f6 100644 --- a/studio/frontend/src/i18n/locales/es.ts +++ b/studio/frontend/src/i18n/locales/es.ts @@ -37,6 +37,8 @@ export const es = { navigation: { newChat: "Nuevo chat", returnToChat: "Volver al chat", + returnToChats: "Volver a {count} chats", + chatGenerating: "Generando", compare: "Comparar", search: "Buscar", hub: "Centro de modelos", diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts index 704eac3fe2..4f2838391f 100644 --- a/studio/frontend/src/i18n/locales/fr.ts +++ b/studio/frontend/src/i18n/locales/fr.ts @@ -37,6 +37,8 @@ export const fr = { navigation: { newChat: "Nouvelle discussion", returnToChat: "Retour à la discussion", + returnToChats: "Retour à {count} discussions", + chatGenerating: "Génération en cours", compare: "Comparer", search: "Rechercher", hub: "Hub de modèles", diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts index c18a86809f..33b827f314 100644 --- a/studio/frontend/src/i18n/locales/hi.ts +++ b/studio/frontend/src/i18n/locales/hi.ts @@ -37,6 +37,8 @@ export const hi = { navigation: { newChat: "नई चैट", returnToChat: "चैट पर लौटें", + returnToChats: "{count} चैट पर लौटें", + chatGenerating: "जनरेट हो रहा है", compare: "तुलना करें", search: "खोजें", hub: "मॉडल हब", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index 9cde9c98ed..978fde6281 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -38,6 +38,8 @@ export const ja = { navigation: { newChat: "新規チャット", returnToChat: "チャットに戻る", + returnToChats: "{count} 件のチャットに戻る", + chatGenerating: "生成中", compare: "比較", search: "検索", hub: "モデルハブ", diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts index a5b7c14940..aa8a4fd47b 100644 --- a/studio/frontend/src/i18n/locales/ko.ts +++ b/studio/frontend/src/i18n/locales/ko.ts @@ -37,6 +37,8 @@ export const ko = { navigation: { newChat: "새 채팅", returnToChat: "채팅으로 돌아가기", + returnToChats: "채팅 {count}개로 돌아가기", + chatGenerating: "생성 중", compare: "비교", search: "검색", hub: "모델 허브", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index 5922e69890..84cd3f945e 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -37,6 +37,8 @@ export const ptBR = { navigation: { newChat: "Novo Chat", returnToChat: "Retornar ao Chat", + returnToChats: "Retornar a {count} chats", + chatGenerating: "Gerando", compare: "Comparar", search: "Buscar", hub: "Hub de modelos", diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts index c680ff1ba9..7725212e3b 100644 --- a/studio/frontend/src/i18n/locales/ru.ts +++ b/studio/frontend/src/i18n/locales/ru.ts @@ -37,6 +37,8 @@ export const ru = { navigation: { newChat: "Новый чат", returnToChat: "Вернуться к чату", + returnToChats: "Вернуться к {count} чатам", + chatGenerating: "Генерация", compare: "Сравнить", search: "Поиск", hub: "Хаб моделей", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index 22f1e06d80..06326ed008 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -37,6 +37,8 @@ export const zhCN = { navigation: { newChat: "新聊天", returnToChat: "返回聊天", + returnToChats: "返回 {count} 个聊天", + chatGenerating: "生成中", compare: "对比", search: "搜索", hub: "模型中心", diff --git a/tests/studio/test_cancel_atomicity.py b/tests/studio/test_cancel_atomicity.py index 391ef043d7..23f340f762 100644 --- a/tests/studio/test_cancel_atomicity.py +++ b/tests/studio/test_cancel_atomicity.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import importlib.util import random import threading from pathlib import Path @@ -107,6 +108,20 @@ _WANTED = { } +def _load_active_generations(): + """The real registry `_TrackedCancel` records runs in. + + Loaded straight off disk rather than imported, so the extracted class runs + against the genuine module without pulling in the whole route package (and + without putting studio/backend on sys.path for the rest of the session). + """ + path = SOURCE_PATH.parents[1] / "state" / "active_generations.py" + spec = importlib.util.spec_from_file_location("studio_active_generations", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _load_registry_module(): chunks = [] for n in _TREE.body: @@ -125,7 +140,7 @@ def _load_registry_module(): and n.target.id in _WANTED ): chunks.append(seg) - mod = {} + mod = {"active_generations": _load_active_generations()} exec( "import threading, time\nfrom typing import Optional\n" + "\n\n".join(chunks), mod, diff --git a/tests/studio/test_deep_research_frontend_contract.py b/tests/studio/test_deep_research_frontend_contract.py index b22d4691a1..c8636518cc 100644 --- a/tests/studio/test_deep_research_frontend_contract.py +++ b/tests/studio/test_deep_research_frontend_contract.py @@ -60,7 +60,9 @@ def test_research_mode_is_single_chat_and_detaches_without_cancel() -> None: assert 'update.event?.event === "reasoning.updated"' in adapter assert "The activity store coalesces these high-frequency events" in adapter assert '{ type: "text" as const, text: report }' in adapter - assert "if (abortSignal.aborted) return" in adapter + # runSignal, not abortSignal: each run gets its own controller, forwarded from the thread + # signal, so one chat's Stop cannot abort a sibling streaming in the background. + assert "if (runSignal.aborted) return" in adapter assert "await autoLoadSmallestModel()" in adapter assert "signal: researchFollowController.signal" in adapter assert "beginExternalResearchFollow(" in adapter diff --git a/tests/studio/test_first_turn_thread_identity.py b/tests/studio/test_first_turn_thread_identity.py new file mode 100644 index 0000000000..49dbc83ef9 --- /dev/null +++ b/tests/studio/test_first_turn_thread_identity.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""A first turn must reach the model adapter with its real thread id. + +assistant-ui binds `unstable_threadId` before the thread is persisted, so a first +turn used to file every run handle under the shared "__default" key. Two of them +overlapping there is unresolvable after the fact: nothing links a run under that +key to the id its thread later receives, so the sidebar showed no spinner and Stop +could not reach either generation. + +The link exists earlier. `append()` tracks `threadListItem.initialize()` by the +user message id, and `createPersistedRunAdapter` already awaits that promise before +invoking the adapter, so the id is known by the time the run starts. These tests pin +that the resolved id is carried through rather than discarded. +""" + +from __future__ import annotations + +import re +from pathlib import Path + + +WORKSPACE = Path(__file__).resolve().parents[2] +PROVIDER = (WORKSPACE / "studio/frontend/src/features/chat/runtime-provider.tsx").read_text( + encoding = "utf-8" +) + + +def test_the_tracked_promise_carries_the_assigned_thread_id(): + # Resolving to void threw the id away, which is what forced the "__default" detour. + assert "Promise\n>();" in PROVIDER + assert re.search( + r"trackRunStartReady\(\s*message\.id,\s*initializeThread\.then\(\(\{ remoteId \}\) => remoteId\),", + PROVIDER, + ), "append() must track the promise that resolves to the persisted thread id" + + +def test_wait_for_run_start_returns_the_id(): + assert re.search( + r"async function waitForRunStartHistoryAppend\([^)]*\): Promise", + PROVIDER, + re.S, + ), "the awaiter must hand back the id it waited for" + assert "return adoptedThreadId;" in PROVIDER + + +def test_the_run_is_given_its_real_thread_id(): + # The whole point: the adapter must not start under the unresolved key when the id is + # already known by the time the await above resolves. + block = re.search( + r"async \*run\(options\) \{.*?const result = adapter\.run\(.*?\);", + PROVIDER, + re.S, + ) + assert block, "createPersistedRunAdapter's run wrapper not found" + body = block.group(0) + assert "const adoptedThreadId = await waitForRunStartHistoryAppend(" in body + assert ( + "!options.unstable_threadId && adoptedThreadId" in body + ), "only fill in the id when assistant-ui had none" + assert "unstable_threadId: adoptedThreadId" in body + + +def test_an_existing_thread_id_is_never_overwritten(): + # A resolved thread already streams under its own id; replacing it would move a running + # chat's handles out from under the sidebar row watching them. + block = re.search( + r"const result = adapter\.run\((.*?)\);", + PROVIDER, + re.S, + ) + assert block + arg = block.group(1) + assert "? { ...options, unstable_threadId: adoptedThreadId }" in arg + assert ": options" in arg diff --git a/tests/studio/test_stop_running_chats_prompt_contract.py b/tests/studio/test_stop_running_chats_prompt_contract.py new file mode 100644 index 0000000000..d3995ec228 --- /dev/null +++ b/tests/studio/test_stop_running_chats_prompt_contract.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Source contracts for the "stop running chats" confirmation. + +The dialog is what a user reads before losing in-flight work, so two things have +to hold: it counts conversations rather than generation handles, and it describes +what confirming actually does. There is no frontend test runner in this repo, so +these read the source the way the other frontend contracts here do. +""" + +from __future__ import annotations + +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parents[2] +FRONTEND = WORKDIR / "studio" / "frontend" / "src" + + +def _read(rel: str) -> str: + path = FRONTEND / rel + assert path.exists(), f"missing source file: {path}" + return path.read_text(encoding = "utf-8") + + +def test_the_prompt_counts_conversations_not_generation_handles(): + # One chat holds several handles while a tool continuation registers its next leg + # before the previous unwinds (active_generations.ActiveGeneration mints one per + # __enter__), so active.count exceeds the deduplicated thread_ids and the dialog + # offered to stop two chats while listing one title. + src = _read("features/chat/utils/confirm-stop-running-chats.ts") + assert "entry.thread_id" in src, "the unnamed entries have to be counted separately" + # The raw handle count survives only for a backend too old to send the entries. + primary = src.index("running.length + unnamed") + fallback = src.index("Math.max(active.count") + assert primary < fallback, "the handle count must be the fallback, not the primary" + + +def test_an_unload_is_not_described_as_a_reload(): + # ejectModel confirms through the same dialog, but confirming calls /unload and + # leaves no model loaded: "Unloading the model reloads the model" and "Stop and + # reload" promised the opposite for the destructive one. + dialog = _read("features/chat/components/stop-running-chats-dialog.tsx") + assert "Stop and unload" in dialog and "Stop and reload" in dialog + assert "leaves no model loaded" in dialog + + runtime = _read("features/chat/hooks/use-chat-model-runtime.ts") + eject = runtime.index('"Unloading the model"') + assert ( + '"unload"' in runtime[eject : eject + 120] + ), "the eject path must ask for the unload wording" + + +def test_the_tts_request_names_its_thread(): + # The audio branch registers its run locally under the thread key, and the backend + # tracker reads payload.thread_id. Omitting it filed the backend entry under no + # thread, so the prompt counted the named local run and the unnamed backend one as + # two requests for a single TTS chat. + src = _read("features/chat/api/chat-adapter.ts") + call = src.index("const result = await generateAudio(") + assert ( + "thread_id: resolvedThreadId" in src[call : call + 600] + ), "the TTS payload must carry the resolved thread id" diff --git a/tests/studio/test_stream_cancel_registration_timing.py b/tests/studio/test_stream_cancel_registration_timing.py index a833006873..5a28432aed 100644 --- a/tests/studio/test_stream_cancel_registration_timing.py +++ b/tests/studio/test_stream_cancel_registration_timing.py @@ -10,6 +10,7 @@ from __future__ import annotations import ast import asyncio +import importlib.util import json import threading import time @@ -256,6 +257,20 @@ _WANTED = { } +def _load_active_generations(): + """The real registry `_TrackedCancel` records runs in. + + Loaded straight off disk rather than imported, so the extracted class runs + against the genuine module without pulling in the whole route package (and + without putting studio/backend on sys.path for the rest of the session). + """ + path = SOURCE_PATH.parents[1] / "state" / "active_generations.py" + spec = importlib.util.spec_from_file_location("studio_active_generations", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _load_registry_module(): chunks = [] for n in _TREE.body: @@ -274,7 +289,7 @@ def _load_registry_module(): and n.target.id in _WANTED ): chunks.append(seg) - mod = {} + mod = {"active_generations": _load_active_generations()} exec("import threading, time\n" + "\n\n".join(chunks), mod) return mod @@ -674,16 +689,18 @@ def test_generate_stream_cancels_backend_on_stream_cancelled_error(): body_src = "\n".join(ast.unparse(stmt) for stmt in sub.body) found_cancel_handler = ( "cancel_event.set()" in body_src - and "backend.reset_generation_state()" in body_src + and "backend.reset_generation_state(cancel_event)" in body_src and any(isinstance(stmt, ast.Raise) and stmt.exc is None for stmt in sub.body) ) if isinstance(sub, ast.Try) and sub.finalbody: final_src = "\n".join(ast.unparse(stmt) for stmt in sub.finalbody) - found_finally_cleanup = ( + # Accumulate: an existence claim, and the cleanup sits in a nested try whose + # own finally only unregisters the swap-gate entry. + found_finally_cleanup = found_finally_cleanup or ( "not completed" in final_src and "not cancel_event.is_set()" in final_src and "cancel_event.set()" in final_src - and "backend.reset_generation_state()" in final_src + and "backend.reset_generation_state(cancel_event)" in final_src and _awaits_to_thread_gen_close(sub) ) @@ -731,11 +748,11 @@ def test_stream_chunks_cancel_branch_resets_backend_state(): ): continue body_src = "\n".join(ast.unparse(s) for s in sub.body) - if "backend.reset_generation_state()" in body_src: + if "backend.reset_generation_state(cancel_event)" in body_src: return raise AssertionError( "stream_chunks `if cancel_event.is_set():` branch must call " - "backend.reset_generation_state() -- matches the existing " + "backend.reset_generation_state(cancel_event) -- matches the existing " "request.is_disconnected() / CancelledError cleanup paths and " "prevents KV-cache drift after cancel-via-POST" ) diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index c2bdbbc915..864941a20a 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -305,7 +305,9 @@ def _find_setup_script() -> Optional[Path]: _PARALLEL_MIN = 1 _PARALLEL_MAX = 64 _PARALLEL_DEFAULT_RUN = 4 # pre-PR hardcoded for `unsloth studio run` -_PARALLEL_DEFAULT_PLAIN = 1 # pre-PR effective for plain `unsloth studio` +# New Chat leaves the previous conversation generating and the admission queue caps decodes at +# the slot count, so at 1 every extra chat queues. _slots_that_fit_on_gpu() may cut it back. +_PARALLEL_DEFAULT_PLAIN = 4 def _resolve_secure(secure: bool, not_secure: bool) -> bool: @@ -1261,8 +1263,7 @@ def studio_default( max = _PARALLEL_MAX, help = ( f"llama-server parallel decode slots ({_PARALLEL_MIN}..{_PARALLEL_MAX}). " - f"Default {_PARALLEL_DEFAULT_PLAIN}; `unsloth studio run` " - f"defaults to {_PARALLEL_DEFAULT_RUN}." + f"Default {_PARALLEL_DEFAULT_PLAIN}." ), ), cloudflare: Optional[bool] = typer.Option(