From c60864955208bca52b63bda9983b3c8fae5e994d Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 28 Jul 2026 04:40:38 -0700 Subject: [PATCH] feat(studio): run chats in parallel in the Chat tab (#7455) * feat(studio): run chats in parallel in the Chat tab New Chat used to cancel whatever the current conversation was generating. It now leaves it running, like switching to the Train or Export tab: the sidebar shows which chats are still going, and Stop is per conversation. Plain `unsloth studio` launched llama-server with one decode slot, so the admission queue serialised every chat regardless of what the UI did. Both entry points now default to the same slot count as `unsloth studio run`. A model swap still ends every running chat, since they all decode on one llama-server. /load and /unload now refuse with 409 and name those chats unless the caller passes force_cancel_active, and the UI asks first. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix(studio): scope the composer tool badge to its own conversation The green "Running Python: ..." badge above the composer read a single global store value, so one chat's tool call showed above every other chat's composer, including a brand-new empty one. Its elapsed counter also restarted at 0 on every thread switch, and a run ending anywhere cleared the badge everywhere. Key the status by thread and store the moment it started, so each conversation shows only its own tool call and the counter resumes rather than restarts. Also adds a test that every conversation gets its own tool sandbox directory, which parallel tool calls depend on. * Fix stalled tool calls while awaiting approval for PR #7455 Three problems, all from the approval prompt behaving as though only one chat could ever run. Arguments were not streamed for a gated call, so the chat stayed blank for as long as the model took to write the payload, which for a large file is minutes. Nothing runs before the decision either way, and the code is what is being approved, so python and terminal now stream their card while gated. render_html stays suppressed: its card renders the payload. The status read "Running ..." with a climbing timer while the call had not started. It now reports that it is waiting for approval, then switches to running once allowed. The admission lease was held across the wait, so four unanswered prompts held all four decode slots and no other chat could start while llama-server sat idle. A parked run keeps its lease but no longer counts against capacity. Measured with four prompts left open: every gated call streamed its code, none reported running, and a fresh chat answered in 0.4s where it previously waited 290s and never did. * Fix duplicated and truncated tool cards for PR #7455 A gated tool call rendered two cards: the provisional one that streams the arguments, plus a second one keyed by the approval id. Only the second ever got its tool_end, so the first spun "Running" for the rest of the chat. Reuse the open part when the approval prompt arrives. The terminal card also showed nothing but a 60-char trigger label, so a long heredoc read as no progress at all. It now renders the command the same way the Python card renders its script, and neither is capped at 10k chars. Both cells moved inside the collapsible, so one chevron hides the code with the output and Copy / Download exist only while the card is open. A card parked on the prompt says so instead of counting up "Running". * Fix review findings on the parallel-chat gate for PR #7455 Backend: - /unload rechecks active generations under the lifecycle gate, like /load, and lets its 409 through the catch-all instead of rewriting it as a 500. - /load gates only once _load_model_impl has decided this is a real reload, so an Apply on the already-loaded model no longer refuses, and the retry it asks for no longer cancels every chat before returning already_loaded. - The direct /v1/responses stream registers in the cancel registry, so a non-forced unload can no longer tear llama-server down under it. - run_server defaults to the same slot count as the CLI. colab.py calls it without the argument, so Colab was still serialising every chat. Frontend: - Cancelling a backgrounded chat aborts its own request rather than only posting a cancel id, which is the only thing that ends an external-provider or audio run. - The model-swap dialog counts local runs only, and falls back to the backend when this tab's map is empty, so a reload or a second tab still gets asked. - Context usage and the diffusion canvas are scoped to the chat that produced them; a compare row reads activity from its member threads. Tests: - The extracted-source cancel harnesses supply the active-generations module, which the tracked-cancel class now depends on. * Fix the swap confirmation scope and cancel timing for PR #7455 A forced load cancelled every chat before the model identifier, GPU selection, training coexistence and download checks had run, so a load that then failed those checks stopped the chats and replaced nothing. The refusal still happens early, but the destructive cancel now sits immediately before the teardown it is paying for, and rechecks under the gate like /unload does. The swap dialog only reconciled with the backend when this tab looked idle, so one local chat was enough to hide a second tab's runs. Confirming then sent force_cancel_active, which cancels every backend run, including the ones the dialog never mentioned. The backend snapshot is now merged in every time, so the dialog names what will actually stop. External-provider runs are never registered there, so the union stays local-only. Also drops the active-generations docstring claim about restoring sidebar spinners, which nothing consumes. * Defer destructive cancels and track every local stream for PR #7455 /unload cancelled the running chats before it had resolved that it unloads anything. A stale model_path, which a second tab produces routinely, killed every chat and then no-opped, leaving the resident model up. It now refuses early and cancels only at each teardown, matching /load. The swap dialog also stopped every chat locally the moment the user confirmed, which threw away the two-phase backend behaviour: a load that then failed identifier resolution, GPU validation or the training guard had already truncated the replies. The backend now owns the cancel. Three local streams decoded on llama-server without registering, so a non-forced unload counted zero generations and tore the server down mid response: /v1/completions streaming, and the plain and server-tool Anthropic streams, the first of which is the default /v1/messages path. Note this makes a non-forced load return 409 during those runs rather than draining quietly, the same trade the /v1/responses fix made. The safetensors tool loop still announced a gated call as running while it waited on a human; only the GGUF loop had been fixed. A source-level parity test now pins both. Also drops stopAllChatThreads, which has no callers left. * Studio: close three load/unload gate races found in review Re-check the in-flight load guard after the stop-running-chats confirm. The confirm always GETs active-generations before its zero-running early-out, so the guard no longer sits atomically ahead of the reservation and two picks in that window both reached performLoad over the same refs. ejectModel had the same shape and gets the same re-check. Reject a sidecar swap immediately before the forced cancel in both load branches. The previous check was back at the top of preflight, so an install reserving during identifier resolution, the tier probe, the training guard or the download check made the post-drain recheck 409 a load whose chats had already been stopped. Enter the Anthropic passthrough's cancel tracker inside its body generator. It was entered eagerly and returned through _sse_streaming_response, which sets no unstarted_cleanup, so a response whose body never started left the run registered forever and 409'd every later non-forced load and unload. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Trim comments across the files this PR touches Tightens the comments and doc blocks in the backend, CLI, tests and frontend files changed by this PR: collapses multi-line explanations to a single line where they still read clearly, and drops the ones the code already says. No code changes, verified by an AST comparison against the previous commit. * Studio: defer the destructive cancel and close two gate gaps Move the forced cancel behind every check that can still reject a swap. The drain now runs first with the runs it is about to cancel discounted, so it waits only for inference the cancel cannot end, then the sidecar check decides, then the cancel fires, then a second drain lets those runs unwind before teardown. A sidecar install reserving during the drain no longer 409s a load whose chats have already been stopped. Track the non-streaming /v1/completions proxy. It was the last local decode path missing from active_generations, so an unload, which runs no drain, tore llama-server down under it and force_cancel_active could not signal it. It now uses the same tracked cancel event and dedicated client as the OpenAI pass-through. Skip the client's preliminary unload while chats are generating and let /load evict at its own post-preflight point instead. Forwarding force_cancel_active there truncated replies before identifier resolution, the GPU and training guards and the download check had run. Keep per-thread context usage so returning to a chat whose background run finished restores its bar instead of leaving it blank until the next turn. Make the running-flag clear run-specific. Every run without a resolved thread id shares the "__default" key, so concurrent compare panes could clear each other's flag and strand a live stop handle. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: register the embeddings proxy with the swap gate /v1/embeddings proxied straight through the pooled client with no tracked cancel event, so it never appeared in active_generations. /unload runs no idle drain, so a concurrent non-forced unload counted zero generations and killed llama-server mid-request, and force_cancel_active had no event to signal. Mirrors the completions proxy: tracked event, dedicated unpooled client closed by a cancel/disconnect watcher, unregister in a nested finally so a close failure cannot leave a phantom generation behind. * Trim comments on the newest changes in this PR Comments only, no code changes: shorten the ones added by the load-gate ordering, embeddings and per-thread usage work down to the same density as the rest of the diff. * Studio: register the legacy generate stream with the swap gate /generate/stream built a cancel event but never entered the tracker, so it was invisible to active_generations. Being in the keep-warm middleware's inference suffixes only covers /load, which drains; /unload does not, so a non-forced unload passed the 409 gate and then blocked on the standard backend's generation lock, and a forced swap had no event to signal. Registered inside the body generator under a nested finally so a teardown failure cannot skip the unregister. The AST contract test asserted the cleanup finally by overwriting its flag per Try node, so a nested try made the last one win. Accumulate instead, which is what the existence claim meant. * Studio: three more swap-gate gaps found in review Register /audio/generate with the gate. TTS holds the model for the whole request and /unload runs no drain, so unregistered a non-forced swap counted zero generations and tore the model down mid-generation; the orchestrator path only waits 15s for the generation lock, which real TTS exceeds. No cancel keys: no backend takes a cancel_event for audio, so the event has no observer and a forced swap still cannot interrupt audio already in flight. Thread the tracked cancel event into the /v1/responses admission wait. It was the only admission caller passing None, so a queued run could not be reached by cancel_all() and a plain /inference/cancel could not stop it at all. Same omission fixed at the upstream send there and on /v1/completions. Let an unforced unload of a stale model path reach the no-op check. Before this PR that request returned 200 and did nothing; the new gate refused it with 409 for a request that reaches no teardown branch. Gate both refusal passes on the disjunction of the route's own teardown conditions, including not is_loaded, so a mid-load GGUF still refuses. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: register the remaining non-streaming decode paths stream defaults to false on all three of these, so they are the ordinary shape of their routes, and each holds a local backend for the whole request. /unload runs no idle drain, so with no registry entry a non-forced swap counted zero generations and tore the backend down mid-request instead of returning 409, and a forced one had no event to signal. Non-streaming /v1/messages: all three helpers ran with an empty registry, since only the streaming siblings were tracked. Registered at the call site because the pass-through takes no cancel_event of its own, and with no cancel keys, matching those siblings. Non-streaming standard chat and audio-input chat: the trackers in this route sit inside their `if payload.stream:` arms, so neither else branch was covered. The GGUF sibling already registers its own non-streaming branch. Each exit is in a finally on the branch's existing try, so the except arms are covered too: a leaked entry 409s every later swap until restart. * Studio: tighten the swap-gate comments Comment-only pass over the newest swap-gate registrations: collapse the multi-line rationales in /unload, the legacy generate stream, audio generation and the non-streaming chat branches, and the matching test preambles, to the shortest form that still carries the reason. No code changes. * Studio: stop the reselect dialog promising a stop that never happens Picking an external provider leaves the local model resident and stops the status poll mirroring it, so reselecting that model showed the stop-chats dialog, and /load then answered already_loaded ahead of its cancel hook. Confirmed with the live backend: the same pick with force_cancel_active set still returned already_loaded and the chat kept streaming. Not stopping those chats is right, since the load never interrupts them, so remove the prompt rather than honour it. Blanket-skipping is unsafe, because the same id and variant with one sampling setting changed is a real reload and 409s, so the branch only fires when a status fetch confirms the resident checkpoint and variant match, and then adopts it without calling /load. Redact native model paths from the active-generations response. Registering /generate/stream recorded backend.active_model_name verbatim, which is an absolute path for a native local model, and this route is the only place that serialises it. Redacting at the response covers every tracker rather than the one that surfaced it. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: keep hydrated context usage in the per-thread map The history loader restores a saved conversation's usage through setContextUsage only, and it runs once per mount, so switching away and back left the bar blank for a hydrated chat even after the per-thread map landed. setContextUsage now writes the value through to the visible thread's own entry and clears that entry when passed null, which covers both hydration call sites and any future writer. * Studio: unblock load cancellation and share unresolved thread keys Run the two stop-loading fast paths ahead of the unload route's pre-gate refusal. _unload_may_evict returns True for exactly the model being cancelled, so the refusal was blocking the branch that cancels a load which has replaced nothing and can interrupt no chat. The client made that unrecoverable: cancelLoading sends the unload without force, drops the result, and its abort never reaches /load, which takes no signal, so the load ran on and could later cancel those chats and swap the model. Nothing else is exempted; an unload that would tear down a serving model matches neither fast path and still 409s. The comment claiming the client lets that 409 surface is corrected, since it discards it. Hold every owner behind a shared thread key. Runs with no resolved thread id share "__default" (concurrent compare panes, since startCompare clears activeThreadId), so a single owner slot let a second run replace the first's token and then delete the shared entry while it was still generating, and the server-cancel map lost the older handle the same way. Both now hold a list, the running and local flags survive until the last owner clears, and stopChatThread stops every handle under the key. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: carry a confirmed swap into the sidecar install, key restored usage by thread Picking a model that needs a newer transformers while chats generate raised the "stop N chats" prompt, but the answer never reached the install that runs before the load: /install-latest-transformers refused on those same chats and took no force flag, so Retry hit the same 409 and nothing in the flow stopped them. Carry force_cancel_active through the consent dialog into the installer. Only the pre-gate fast path is skipped: the recheck under the lifecycle gate still has to pass, so an unconfirmed caller is refused as before. The cancel runs last inside the gate, after every check that can still reject the install, and the drain behind it is bounded since it holds the gate and the sidecar reservation. Also key restored context usage by the thread the loader read. history.load() captures remoteId before two awaited round trips, so a switch inside that window filed one thread's usage under another and setActiveThreadId kept re-applying it. Preserve sibling owners when a run key is cleared without an owner: the image rejection gate now uses its own token, and the reducer leaves owned runs alone. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: bound the post-cancel drains, and make cancellation reach the paths that ignored it A forced swap cancels the chats it interrupts, then waits for them to unwind. That wait had no deadline while holding the lifecycle gate, and TTS on the subprocess backend observes no cancel event at all, so one audio generation could pin every load, unload and new request for its whole duration. Bound both post-cancel drains. Pre-cancel drains stay unbounded: the swap can still be refused there, so shortening them would weaken what they protect. /unload had the opposite problem and no drain at all, cancelling and tearing down on the next line, which turned a clean stream end into a dropped connection. Give it the same bounded wait, gated on the cancel having cancelled something so an idle Eject pays nothing. Make the cancel actually land where it can. GGUF TTS now takes a cancel_event and a watcher closes its client to break the blocking POST. The Anthropic non-streaming pass-through did the same thing the completions and embeddings paths used to: register with the gate, then run both POSTs on the pooled client that cannot be closed. It now uses a per-request client like they do. Also: park and unpark the admission queue the reservation actually holds, since queues are keyed by base_url and a reload mints a new port; key tool output by remoteId on both sides, so the first turn of a New Chat stops writing under one key and reading another; and give tool status a run owner, so a finishing run cannot blank the badge a concurrent one is still showing. Clamp --parallel to 1 on a llama-server without --kv-unified. The new default of 4 would otherwise split -c four ways on such a build, quartering the context window for a feature it cannot serve. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: scope a chat's Stop to its own generation, and clear the way before a confirmed install Safetensors generation is serialized on _gen_lock and the worker has a single cancel event, so a chat still queued on that lock owns no generation. Its Stop handler called reset_generation_state() anyway, which set the shared event and ended whichever conversation was actually running. Parallel chats is what makes that reachable. _generate_inner now records its cancel_event as the current holder once it takes the lock, and reset_generation_state drops a reset from anyone else. Every route call site passes its own request event. A reset with no event stays global, so unload and model switch cannot leave a generation alive, and a reset while nothing runs still resets, so an error path before generation is not a no-op. The other two backends take the argument too, or the standard one raises TypeError on every cancel. The sidecar install had the mirror of the /load ordering problem: it cancelled the chats first and drained second, so an unrelated counted request the cancel cannot reach (a count_tokens, say) was still there for the recheck, which then refused an install that had already stopped every chat for nothing. Drain the unreachable remainder first, discounting the registered chats, then cancel. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: close the windows the previous round's fixes left open Three follow-ups, two of them holes in the fixes just before them. The worker claim went in after _send_cmd, so the command was already running unclaimed and a queued chat's Stop in that window still reset it. Claim first, with the send inside the same try, so a failed send releases it too. Tool status kept one entry per key with an owner. That stops a foreign clear but not an overwrite: under the shared unresolved-thread key the second run replaced the first's entry, and its own clear then removed the only one while the first tool was still running. Keep per-run entries and render the newest. /unload gated its drain on having cancelled something, so a request that passed the keep-warm middleware but had not reached its tracker yet was invisible to it and the teardown landed on an already-admitted request. Drain on the middleware count instead, which covers that window as well as the cancelled runs, then re-cancel whatever registered while waiting. Bounded, not a refusal: an unload is deliberate, and on expiry it proceeds exactly as before. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Trim the parallel-chats comments to their reasons Compress the multi-line rationales added by this branch into shorter forms and drop restatements of the code below them. The reasons behind the drain bounds, the deferred cancel, the per-request generation ownership and the thread-scoped tool and usage keys are kept, just said in fewer lines. * Studio: own the worker per generation, and make a resumed chat requeue for its slot Ownership was a single lock holder, so dispatched runs (compare mode bypasses _gen_lock by design) never claimed it and the guard fell straight through to the global reset: a Stop on one of them ended its siblings. Track the generations actually running instead, claimed before the send and released in the same finally on both paths. A reset still proceeds when nothing is running, so an error path ahead of generation is not swallowed. park() hands the freed slot to a waiter, so a chat resuming from a tool approval could take it back while that waiter was still decoding, putting two holders on a one-slot server and sending the resumed tool loop past the admission limit. unpark_async waits for room; the plain unpark stays for a holder tearing down, which will not decode again. Audio only observed its cancel event on a forced swap. An explicit Stop just aborts the fetch, and this route has no cancel id, so llama-server ran on to the request timeout after the chat reported it stopped. Watch the disconnect. Also read tool status by remoteId, matching the key the adapter writes and the fix already made for tool output, and stop an unresolved run from writing its usage into whichever conversation the user moved to. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: let only the generation the worker is running speak for it, and hold a slot for a resuming chat The ownership list recorded admission, but the subprocess runs generations one at a time, so a dispatched request queued behind another counted as an owner and its Stop signalled the shared cancel event, ending the request that was actually running. Keep admission for release bookkeeping and gate ownership on execution instead, promoted when the worker first answers that request. Nothing executing still permits a reset, so an error path ahead of generation is not swallowed. The worker has one cancel event and no per-request cancellation, so this decides who may pull the lever rather than making the lever per-request. A resuming chat also polled for a slot it could never see: release() grants to the next waiter under the same lock, so later arrivals overtook an approved chat indefinitely. A pending unpark now reserves the next slot and they queue behind it. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: cover the prefill window, and keep a first turn's tool output readable Gating worker ownership on execution left the interval between the send and the first response uncovered: nothing is executing then, and the empty case admitted anyone, so a queued chat's Stop still ended the one in prefill. Split the empty case. Nothing claimed at all still permits a reset, so an error path ahead of generation is not swallowed; claimed but unanswered resolves to the oldest claim, which is what a FIFO command queue is working on. Putting both sides of the tool-output scope on remoteId left the first turn of a New Chat writing under the unresolved scope for its whole life while the readers recomputed the moment the autosave assigned an id, so the card blanked mid-run. The readers now fall back to the unresolved scope, which only an unpersisted first turn can occupy. * Studio: order the parked approvals, and tie a worker claim to its enqueue The reservation added for admission fairness was a bare count, so every approved holder counted against every other: park two chats, approve both, and once the last decoder released, nothing could ever satisfy the check again. That is a deadlock where the problem it fixed was only unfairness. Make it a FIFO ticket so a pending unpark blocks the ones behind it and no others. _owns_worker reads claim order to decide which request the worker is prefilling, which only holds if claiming and enqueuing cannot interleave. Hold one lock across both on the dispatched and the locked path. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: follow a first turn's run when its thread gets an id, and key the denoising canvas per chat A run started before its thread existed filed every handle under "__default". Nothing moved them once autosave assigned the real id, so the sidebar row showed no spinner and Stop could not reach the generation, which kept holding a slot. adoptDefaultThreadRun re-keys the run maps onto the real id from the thread adapter's initialize(), where the id first exists; anything already filed under that id wins, since that is a later run. The adapter captures its key once at run start, so it now resolves the live key per use through runKeyForOwner, looking its own serverCancel up in the owner map. Without that the migrated entries are stranded and the spinner never clears. The denoising canvas was one global slot, so two diffusion chats overwrote each other and the ownership tag then hid the visible preview until that thread emitted again. It is now activeDiffusionCanvasByThreadId, written and cleared per thread, and the frame no longer carries a threadId of its own. The bubble reads threadListItem.remoteId, dropping the dead threadListItem.id arm: the writer tags unstable_threadId, which is exactly remoteId. Two existing backend tests needed the same treatment. _bare_orchestrator skips __init__, so it now sets the claim bookkeeping the worker ownership check reads. The Anthropic passthrough gate test anchored on comment prose that a rewrap had broken; it anchors on the code instead. * Studio: hand the worker over cleanly between generations, and stop unresolved runs sharing each other's state Worker ownership moved off the consumer and onto the dispatcher. Consumers read their mailbox whenever they get around to it, so a request whose gen_done had been routed still owned the worker while the next one ran, and a late Stop for it cancelled that one. The dispatcher is the only place responses arrive in the order the worker produced them: it now retires a request at its terminal response and promotes the next one, and answering a request makes it the sole executor, since the subprocess runs one generation at a time. reserve()'s immediate path ignored the unpark tickets that _grant_waiters_locked already honours, so a request arriving between a slot freeing and an approved chat's next poll took it, repeatedly. It applies the same reservation now. Three places let concurrent first turns share state through the "__default" key. Nothing links a run filed there to the id its thread later receives, so rather than guess, each now declines when the key is ambiguous: adoption only re-keys a lone run, the composer badge only claims a lone status, and the tool-output fallback only applies to a thread that is still running. That leaves two concurrent first turns where they were before adoption existed instead of handing one thread the other's handles. A first turn's usage was never filed, because its key stayed null for the whole run while autosave moved activeThreadId to the real id, so the context bar went blank after the first reply. It resolves the adopted key like the cleanup handles do. Cancelling a forced load left the UI with no model: the previous one stays resident until /load's teardown, and the cancel path cleared the checkpoint without rolling back. It now resyncs from the backend, which is right whether or not the load got that far. The sidecar install drain is weighted 1:4 rather than halved, total unchanged. Only the second half benefits from patience, and cutting it short refused installs whose chats had already been stopped for nothing. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: give a first turn its real thread id before the run starts A first turn filed every run handle under a shared unresolved key because assistant-ui binds unstable_threadId before the thread is persisted. Two of them overlapping there is unresolvable afterwards, and the last round's migration could only decline rather than guess, which left neither sidebar row showing its run. The id is available earlier than I claimed. append() already tracks threadListItem.initialize() by the user message id, and createPersistedRunAdapter already awaits that promise before invoking the adapter, so the thread is persisted by the time the run begins. It was only being discarded: the tracked promise resolved to void. It now resolves to the assigned id, and the wrapper hands it to the adapter when assistant-ui had none. An id that is already set is never replaced, since that would move a running chat's handles out from under the row watching them. The existing unresolved-key guards stay as a safety net but should no longer carry weight. The sidebar counted running thread ids rather than rows, so one compare conversation read as two chats. It folds ids into rows through the same threadIds the row spinner uses, and still counts a running id that matches no row. _TrackedCancel always registered kind="chat", so an embeddings or raw completions request appeared in the model-swap prompt as an unnamed conversation and confirming cancelled it while calling it a chat. The non-conversation routes now pass their own kind, and the prompt says "requests" whenever the snapshot is not all chats. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: withhold the shared worker cancel from a request the worker has left Moving ownership to the dispatcher fixed reset_generation_state, but the token loop signals the shared worker event directly and did not carry the same rule. A dispatched consumer runs with mark_started off and can still be draining tokens buffered before its gen_done was routed, so stopping it there ended whichever request the worker had started next. It now signals only when _owns_worker agrees, the same predicate reset_generation_state uses. The local drain and return are unconditional, since those touch nothing but this stream. The remaining _cancel_generation callers are deliberately global: subprocess shutdown, the pre-load kill and unload_model. * Studio: add the AGPL-3.0 header to the first-turn identity test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: stop the dispatcher and a _gen_lock stream fighting over the response queue Nothing stopped the dispatcher starting under a _gen_lock generation, so once compare was opened while an ordinary chat was still streaming, both consumed _resp_queue and whichever response the dispatcher took without a mailbox was dropped, gen_done included. That chat truncated or hung. This PR is what makes it reachable, since navigating into compare no longer ends the chat behind it. Delaying the dispatcher would serialise compare behind whatever chat happens to be streaming, so the direct readers get a mailbox instead. _direct_reader returns a reader, a cancel drain and a release, and files the mailbox under _direct_mailboxes rather than _mailboxes, which means "compare requests are in flight" to the unload and distributed paths and must not count an ordinary chat. Both directions close. The dispatcher finds the direct reader's mailbox instead of dropping. And this reader can already be blocked on the queue when a compare request's dispatcher starts, so a response that is not ours goes to its own mailbox rather than being consumed, which would have corrupted the chat and hung the pane. All three _gen_lock readers use it, and the cancel drain goes through it too. The sidebar's return target still picked a raw pane id while the count grouped by row, and /chat addresses compare with `compare`, not `thread`. It resolves through the same items now, so a running compare row returns to its pair. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: keep worker ownership honest across audio, API traffic and a replaced worker The audio-input send got a mailbox last round but stayed unclaimed, so a compare request queued behind it looked like the oldest owner and stopping that queued request signalled the shared event into the audio chat. It claims under the send lock and releases in the finally, like _generate_inner. Ownership is keyed on cancel-event identity with nothing tying it to a worker generation, so a consumer still blocked on its mailbox when the process was replaced stayed recorded as the executor, and a generation on the fresh worker could not be stopped. _shutdown_subprocess clears that state once the process is confirmed dead, mailboxes included: nothing routes to them again, and a stale one reads as compare activity to the unload path. Not on the survived-SIGKILL path, which keeps its handle on purpose. The four public /v1/messages trackers were registering as chats. The distinction is a Studio thread, not the protocol, and those branches already say "No thread_id: public API surface" while the Studio path passes payload.thread_id separately. They carry their own kind now, so the swap prompt stops calling an external request a chat. The swap confirmation still counted raw pane ids, so a compare conversation asked to stop two chats and listed its title twice. It folds panes onto pairId and lowers the count by what it collapsed, leaving a first turn the backend can count but not name. Deep Research set runningByThreadId but registered no server-cancel handle, and that map is how Stop, archive and delete reach a thread that is no longer active. Leaving the outgoing thread running is this PR's doing, so the run was left unreachable while its supervisor kept working against a conversation the user could delete. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: tighten the parallel-chats comments * Studio: replay a Deep Research stop that arrived before the run existed The handle is registered before createResearchRun resolves because the thread can be stopped while that request is in flight, but it had no id to act on and dropped the stop. The supervisor then followed a run the user had already stopped, archived or deleted. It latches instead: a stop with no id yet sets a flag, and the adapter replays it against the id the moment creation returns rather than starting to follow. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: fix worker ownership on a raced reroute, and the stop-chats prompt Four review findings on the parallel-chats work, all reproduced first. - _direct_reader hands a foreign response to its own mailbox, but skipped the ownership move the dispatcher makes. A _gen_lock reader already blocked on resp_queue can beat the compare dispatcher to that request's first response, and the compare consumer opts out of marking, so nothing promoted it: the direct request stayed the recorded executor, its late reset cancelled the compare generation, and the compare chat's own Stop was ignored. - A chat stopped while queued on _gen_lock was still claimed and sent once the lock freed. Cancellation is only checked on a token, so a long prefill, or a generation reaching gen_done without one, occupied the worker after Stop. Same hole in the audio-input path, which shares the lock. - The stop-chats prompt counted generation handles, not conversations. One chat holds several while a tool continuation registers its next leg before the previous unwinds, so it offered to stop two chats and listed one title. - Ejecting a model confirms through that dialog, which told the user "Unloading the model reloads the model" and offered "Stop and reload". Confirming calls /unload and leaves nothing loaded. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Studio: name the TTS run's thread so the stop prompt counts it once The audio branch registers its run locally under the thread key but sent no thread_id, so the backend tracker filed the same generation under no thread. The stop-chats prompt then had a named local run and an unnamed backend one and, since e8e7594 started adding unnamed entries to the named ones, counted a single TTS chat as two requests. The backend already reads payload.thread_id, so sending it lines both registries up on the same run. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- studio/backend/core/inference/inference.py | 9 +- .../backend/core/inference/llama_admission.py | 162 +- studio/backend/core/inference/llama_cpp.py | 113 +- .../backend/core/inference/mlx_inference.py | 3 +- studio/backend/core/inference/orchestrator.py | 327 +- .../core/inference/safetensors_agentic.py | 25 +- .../core/inference/tool_loop_controller.py | 13 + studio/backend/core/inference/tools.py | 16 + studio/backend/models/inference.py | 22 + studio/backend/routes/inference.py | 1464 ++++++--- studio/backend/run.py | 22 +- studio/backend/state/active_generations.py | 146 + .../backend/tests/test_active_generations.py | 2635 +++++++++++++++++ .../backend/tests/test_anthropic_admission.py | 18 +- .../test_anthropic_passthrough_respawn.py | 10 +- .../test_inference_dispatcher_resilience.py | 66 + studio/backend/tests/test_llama_admission.py | 219 ++ .../backend/tests/test_llama_cpp_tool_loop.py | 44 + .../backend/tests/test_openai_auto_switch.py | 24 +- .../tests/test_openai_tool_passthrough.py | 45 +- .../tests/test_orchestrator_unload_cancel.py | 320 ++ .../backend/tests/test_passthrough_healing.py | 13 +- .../tests/test_safetensors_tool_loop.py | 24 + .../tests/test_sf_client_tools_passthrough.py | 2 +- .../test_shutdown_preserves_live_worker.py | 10 + .../tests/test_tool_sandbox_per_thread.py | 80 + studio/backend/tests/test_tool_xml_strip.py | 3 +- studio/frontend/src/app/routes/__root.tsx | 3 + .../frontend/src/components/app-sidebar.tsx | 83 +- .../components/assistant-ui/markdown-text.tsx | 4 + .../src/components/assistant-ui/thread.tsx | 65 +- .../assistant-ui/tool-code-cell.tsx | 219 ++ .../components/assistant-ui/tool-group.tsx | 17 +- .../assistant-ui/tool-live-output.tsx | 8 +- .../assistant-ui/tool-ui-python.tsx | 204 +- .../assistant-ui/tool-ui-terminal.tsx | 81 +- studio/frontend/src/components/ui/spinner.tsx | 19 +- .../src/features/chat/api/chat-adapter.ts | 206 +- .../src/features/chat/api/chat-api.ts | 21 + .../frontend/src/features/chat/chat-page.tsx | 13 +- .../components/stop-running-chats-dialog.tsx | 92 + .../chat/hooks/use-chat-model-runtime.ts | 113 +- .../chat/hooks/use-chat-sidebar-items.ts | 12 +- studio/frontend/src/features/chat/index.ts | 5 + .../src/features/chat/runtime-provider.tsx | 80 +- .../chat/stores/chat-runtime-store.ts | 331 ++- .../stores/stop-running-chats-dialog-store.ts | 69 + .../src/features/chat/tool-approval.ts | 18 + .../src/features/chat/tool-output-scope.ts | 51 +- .../frontend/src/features/chat/types/api.ts | 8 + .../chat/utils/confirm-stop-running-chats.ts | 100 + .../chat/utils/prompt-queue-boundary.ts | 13 +- .../features/chat/utils/stop-chat-thread.ts | 39 + .../api/transformers-upgrade-api.ts | 9 +- .../hooks/use-transformers-upgrade-consent.ts | 5 + .../transformers-upgrade-dialog-store.ts | 12 +- studio/frontend/src/i18n/locales/ar.ts | 2 + studio/frontend/src/i18n/locales/de.ts | 2 + studio/frontend/src/i18n/locales/en.ts | 2 + studio/frontend/src/i18n/locales/es.ts | 2 + studio/frontend/src/i18n/locales/fr.ts | 2 + studio/frontend/src/i18n/locales/hi.ts | 2 + studio/frontend/src/i18n/locales/ja.ts | 2 + studio/frontend/src/i18n/locales/ko.ts | 2 + studio/frontend/src/i18n/locales/pt-br.ts | 2 + studio/frontend/src/i18n/locales/ru.ts | 2 + studio/frontend/src/i18n/locales/zh-CN.ts | 2 + tests/studio/test_cancel_atomicity.py | 17 +- .../test_deep_research_frontend_contract.py | 4 +- .../studio/test_first_turn_thread_identity.py | 76 + ...test_stop_running_chats_prompt_contract.py | 63 + .../test_stream_cancel_registration_timing.py | 29 +- unsloth_cli/commands/studio.py | 7 +- 73 files changed, 7007 insertions(+), 946 deletions(-) create mode 100644 studio/backend/state/active_generations.py create mode 100644 studio/backend/tests/test_active_generations.py create mode 100644 studio/backend/tests/test_tool_sandbox_per_thread.py create mode 100644 studio/frontend/src/components/assistant-ui/tool-code-cell.tsx create mode 100644 studio/frontend/src/features/chat/components/stop-running-chats-dialog.tsx create mode 100644 studio/frontend/src/features/chat/stores/stop-running-chats-dialog-store.ts create mode 100644 studio/frontend/src/features/chat/tool-approval.ts create mode 100644 studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts create mode 100644 studio/frontend/src/features/chat/utils/stop-chat-thread.ts create mode 100644 tests/studio/test_first_turn_thread_identity.py create mode 100644 tests/studio/test_stop_running_chats_prompt_contract.py diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py index 563a6732a1..0af37e627f 100644 --- a/studio/backend/core/inference/inference.py +++ b/studio/backend/core/inference/inference.py @@ -2281,8 +2281,13 @@ class InferenceBackend: except Exception as e: logger.warning(f"Could not fully reset model state for {model_name}: {e}") - def reset_generation_state(self): - """Reset any cached generation state to prevent hanging after errors""" + def reset_generation_state(self, caller_cancel_event = None): + """Reset any cached generation state to prevent hanging after errors + + ``caller_cancel_event`` is accepted for signature parity with the + orchestrator, which uses it to drop a reset from a request that never + started. Nothing here cancels a live generation, so it is unused. + """ try: # Clear cached state for ALL loaded models for model_name in self.models.keys(): diff --git a/studio/backend/core/inference/llama_admission.py b/studio/backend/core/inference/llama_admission.py index 1a9ae04b0e..db9a5d8ce4 100644 --- a/studio/backend/core/inference/llama_admission.py +++ b/studio/backend/core/inference/llama_admission.py @@ -214,7 +214,7 @@ class _Waiter: class LlamaAdmissionLease: - __slots__ = ("_queue", "_slot", "_released", "_release_lock") + __slots__ = ("_queue", "_slot", "_released", "_release_lock", "_parked") def __init__( self, @@ -225,20 +225,88 @@ class LlamaAdmissionLease: self._slot = slot self._released = False self._release_lock = threading.Lock() + self._parked = False @property def slot(self) -> Optional[int]: """Pool slot this lease holds, or None when admission is disabled.""" return self._slot + def park(self) -> None: + """Hand the slot back while this holder waits on something off the GPU. + + A run stopped on a tool approval prompt is not decoding, so holding its + slot would let unanswered prompts fill the pool while llama-server idles. + The lease itself stays valid: releasing it after a park is still correct. + """ + queue = self._queue + slot = None + with self._release_lock: + if queue is None or self._released or self._parked: + return + self._parked = True + slot, self._slot = self._slot, None + queue.park(slot) + + def unpark(self) -> None: + """Drop the parked state without reclaiming a slot. + + For a holder that is tearing down: it will not decode again. Resuming + holders must use ``unpark_async``, which waits for a slot instead of + going back to llama-server past the admission limit. + """ + with self._release_lock: + if not self._parked: + return + self._parked = False + if self._queue is not None: + self._queue.unpark() + + async def unpark_async( + self, + *, + cancel_event = None, + poll_s: float = 0.02, + ) -> None: + """Take a slot back, waiting until the pool has room. + + ``park`` gave the slot to a waiter, so by the time the user answers the + prompt someone else may be decoding in it. Resuming regardless put two + holders on a one-slot server. Gives up if the caller is cancelled, since + the holder is then leaving anyway and must not be stuck here. + """ + queue = self._queue + if queue is None or not self._parked: + return + slot = await queue.acquire_parked_slot(cancel_event = cancel_event, poll_s = poll_s) + stranded = None + with self._release_lock: + # release() may have run during the wait; it clears the flag and does + # the unpark itself, so only the caller that clears it here repeats one. + parked, self._parked = self._parked, False + if self._released: + # Released while waiting: this lease will never hand the slot + # back, so return it here rather than strand it for good. + stranded = slot + else: + self._slot = slot + if parked: + queue.unpark() + if stranded is not None: + queue.release(stranded) + def release(self) -> None: queue = None + parked = False with self._release_lock: if self._released: return self._released = True queue = self._queue + parked, self._parked = self._parked, False if queue is not None: + if parked: + queue.unpark() queue.release(self._slot) async def __aenter__(self) -> "LlamaAdmissionLease": @@ -338,7 +406,18 @@ class LlamaAdmissionQueue: set to 0. See ``LlamaAdmissionConfig.queue_limit``. """ - __slots__ = ("key", "_lock", "_capacity", "_free", "_in_use", "_held", "_waiters") + __slots__ = ( + "key", + "_lock", + "_capacity", + "_free", + "_in_use", + "_held", + "_waiters", + "_parked", + "_unpark_tickets", + "_unpark_seq", + ) def __init__(self, key: str): self.key = key @@ -351,6 +430,13 @@ class LlamaAdmissionQueue: self._in_use = 0 self._held = 0 self._waiters: Deque[_Waiter] = deque() + # Holders parked on a tool approval prompt. They hold no slot, so this only + # keeps the queue off the idle-eviction list while they are away. + self._parked = 0 + # FIFO tickets for holders resuming from a park (see acquire_parked_slot). A + # bare count deadlocked: every approved holder blocked every other one. + self._unpark_tickets: Deque[int] = deque() + self._unpark_seq = 0 def _resize_pool_locked(self, capacity: int) -> None: # Slots past a shrunk capacity retire when their holder releases them. @@ -359,13 +445,15 @@ class LlamaAdmissionQueue: self._capacity = capacity self._free = [slot for slot in range(capacity) if not self._in_use >> slot & 1] - def _can_admit_locked(self) -> bool: + def _can_admit_locked(self, reserved: int) -> bool: # Slots still held above a shrunk capacity keep occupying the backend, so # count every held slot against the ceiling, not just the ids below it. - return bool(self._free) and self._held < self._capacity + # ``reserved`` holds slots back for approved holders waiting to resume; + # without it a stream of new arrivals took the next slot, forever. + return bool(self._free) and (self._held + reserved) < self._capacity - def _take_slot_locked(self) -> Optional[int]: - if not self._can_admit_locked(): + def _take_slot_locked(self, reserved: int) -> Optional[int]: + if not self._can_admit_locked(reserved): return None slot = self._free.pop() self._in_use |= 1 << slot @@ -386,7 +474,7 @@ class LlamaAdmissionQueue: self._resize_pool_locked(capacity) self._grant_waiters_locked() if not self._waiters: - slot = self._take_slot_locked() + slot = self._take_slot_locked(len(self._unpark_tickets)) if slot is not None: # No snapshot here: callers read it through snapshot_now(), # which re-reads the queue, so building one per admitted @@ -425,6 +513,58 @@ class LlamaAdmissionQueue: self._release_slot_locked(slot) self._grant_waiters_locked() + def park(self, slot: Optional[int]) -> None: + """Return a parked holder's slot to the pool. See ``LlamaAdmissionLease.park``.""" + with self._lock: + self._parked += 1 + self._release_slot_locked(slot) + self._grant_waiters_locked() + + def unpark(self) -> None: + with self._lock: + if self._parked > 0: + self._parked -= 1 + + async def acquire_parked_slot( + self, + *, + cancel_event = None, + poll_s: float = 0.02, + ) -> Optional[int]: + """Wait for a slot for a holder resuming from a park, None if cancelled. + + Ordered by ticket rather than counted, so approvals resume in the order + they came back: counting them made every approved holder block every + other one, and with nothing decoding that never resolved. + """ + with self._lock: + self._unpark_seq += 1 + ticket = self._unpark_seq + self._unpark_tickets.append(ticket) + try: + while True: + with self._lock: + ahead = 0 + for queued in self._unpark_tickets: + if queued == ticket: + break + ahead += 1 + # Only the approvals ahead of this one hold slots back from it. + slot = self._take_slot_locked(ahead) + if slot is not None: + return slot + if cancel_event is not None and cancel_event.is_set(): + return None + await asyncio.sleep(poll_s) + finally: + with self._lock: + try: + self._unpark_tickets.remove(ticket) + except ValueError: + pass + # This ticket was holding a slot back from the wait line. + self._grant_waiters_locked() + def cancel(self, waiter: _Waiter) -> None: lease_to_release = None with self._lock: @@ -455,15 +595,17 @@ class LlamaAdmissionQueue: def is_idle(self) -> bool: with self._lock: self._prune_waiters_locked() - return self._in_use == 0 and not self._waiters + # A parked holder owns no slot but is coming back to this queue, so + # evicting it here would resume it against a fresh 1-slot pool. + return self._in_use == 0 and not self._waiters and not self._parked def _grant_waiters_locked(self) -> None: # Dead waiters are skipped as they are popped, so no prune is needed here. - while self._waiters and self._can_admit_locked(): + while self._waiters and self._can_admit_locked(len(self._unpark_tickets)): waiter = self._waiters.popleft() if waiter.cancelled or waiter.future.done(): continue - slot = self._take_slot_locked() + slot = self._take_slot_locked(len(self._unpark_tickets)) lease = LlamaAdmissionLease(self, slot) waiter.granted_lease = lease try: diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c83d3696a8..a23501a6eb 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -98,6 +98,7 @@ from core.inference.tool_call_parser import ( from core.inference.tool_loop_controller import ( ToolLoopController, append_deferred_nudges, + awaiting_approval_status, tool_event_provenance, ) from state.tool_approvals import ( @@ -6551,6 +6552,25 @@ class LlamaCppBackend: binary = self._find_llama_server_binary() is_vulkan_backend = self._is_vulkan_backend(binary) + # Without --kv-unified an explicit --parallel N splits -c into windows of -c/N, so on a + # build lacking the flag the default of 4 would quarter every context window for a + # feature it cannot serve: fall back to one slot. Ahead of the KV estimates so the + # fit matches what launches. + if ( + n_parallel > 1 + and binary + and not self.probe_server_capabilities(binary).get("supports_kv_unified") + ): + logger.warning( + "llama-server at %s has no --kv-unified, so %d parallel slots would " + "split the context window %d ways. Using 1 slot instead; update " + "llama.cpp to run chats in parallel.", + binary, + n_parallel, + n_parallel, + ) + n_parallel = 1 + # ── Vulkan-ordinal preflight (BEFORE the Phase 1 kill) ──────── # An explicit Vulkan pin the ggml probe never enumerated cannot be honored. # Validate it ABOVE the kill so an invalid selection leaves the live model @@ -11101,6 +11121,7 @@ class LlamaCppBackend: from core.inference.tools import ( build_rag_autoinject, execute_tool, + has_text_only_provisional_card, is_always_safe_tool, is_high_risk_tool_call, ) @@ -11527,6 +11548,9 @@ class LlamaCppBackend: permission_mode == "auto" and is_always_safe_tool(current_name) ) + # A text-preview card still streams while gated; + # hiding it blanks the chat. + and not has_text_only_provisional_card(current_name) ) # Keep small-argument tools on the normal path. _args_len = len( @@ -11628,20 +11652,27 @@ class LlamaCppBackend: # TEXT call to a provisional card. Gated on an enabled-name # sniff + size floor so prose/small calls spawn no pane; id # matches the first call so the final tool_start reconciles. - if ( - not has_structured_tc - and not _confirm_gated_iteration - and _text_args_call_start >= 0 - ): + if not has_structured_tc and _text_args_call_start >= 0: if not _text_args_id: _call_text = content_accum[_text_args_call_start:] _sniffed = _sniff_text_tool_name( _call_text, _enabled_tool_names ) - if _sniffed and ( - _sniffed == "render_html" - or len(_call_text) - >= _PROVISIONAL_ARGS_MIN_CHARS + # Structured-path rule: gated calls + # stream only from a text-preview card. + if ( + _sniffed + and not ( + _confirm_gated_iteration + and not has_text_only_provisional_card( + _sniffed + ) + ) + and ( + _sniffed == "render_html" + or len(_call_text) + >= _PROVISIONAL_ARGS_MIN_CHARS + ) ): _text_args_id = "call_0" _text_args_name = _sniffed @@ -12230,18 +12261,31 @@ class LlamaCppBackend: start_event["awaiting_confirmation"] = needs_confirm try: - yield {"type": "status", "text": decision.status_text} + # Gated calls are not running yet; a "Running ..." badge + # counting up while it waits on a human reads as a hang. + yield { + "type": "status", + "text": ( + awaiting_approval_status(decision.tool_name) + if needs_confirm + else decision.status_text + ), + } yield start_event - if ( - decision_slot is not None - and wait_tool_decision( + _decision = ( + wait_tool_decision( decision_slot, approval_id, cancel_event = cancel_event, ) - == "deny" - ): + if decision_slot is not None + else None + ) + if _decision is not None and _decision != "deny": + # Approved: now it really is running. + yield {"type": "status", "text": decision.status_text} + if _decision == "deny": decision_slot = None resolved_provisional_tool_call_ids.add(decision.tool_call_id) yield { @@ -12809,10 +12853,15 @@ class LlamaCppBackend: min_p: float = 0.0, max_new_tokens: int = 2048, repetition_penalty: float = 1.1, + cancel_event: Optional[threading.Event] = None, ) -> tuple: """ Generate TTS audio via llama-server /completion + codec decode. Returns (wav_bytes, sample_rate). + + ``cancel_event`` lets a Stop or a forced model swap end the request: the + decode is one blocking POST, so a watcher closes the client out from under + it rather than polling. Raises RuntimeError once cancelled. """ if audio_type not in self._TTS_PROMPTS: raise RuntimeError(f"GGUF TTS does not support '{audio_type}' codec.") @@ -12834,15 +12883,47 @@ class LlamaCppBackend: if need_ids: payload["n_probs"] = 1 + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") + with httpx.Client( timeout = httpx.Timeout(300, connect = 10), headers = self._auth_headers, trust_env = False, ) as client: - resp = client.post(f"{self.base_url}/completion", json = payload) + finished = threading.Event() + watcher: Optional[threading.Thread] = None + if cancel_event is not None: + + def _close_when_cancelled() -> None: + while not finished.wait(0.05): + if cancel_event.is_set(): + # Closing mid-request makes the blocking post raise + # httpx.RequestError, the only way out of it. + with contextlib.suppress(Exception): + client.close() + return + + watcher = threading.Thread(target = _close_when_cancelled, daemon = True) + watcher.start() + try: + resp = client.post(f"{self.base_url}/completion", json = payload) + except httpx.RequestError: + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") from None + raise + finally: + finished.set() + if watcher is not None: + watcher.join(timeout = 0.5) if resp.status_code != 200: raise RuntimeError(f"llama-server returned {resp.status_code}: {resp.text}") + # The codec decode below is GPU work with no interruption point, so check here: + # cancelling after this only wastes the decode it cannot stop. + if cancel_event is not None and cancel_event.is_set(): + raise RuntimeError("Audio generation cancelled") + data = resp.json() token_ids = ( [p["id"] for p in data.get("completion_probabilities", []) if "id" in p] diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py index d19c67a01a..2b300a32b1 100644 --- a/studio/backend/core/inference/mlx_inference.py +++ b/studio/backend/core/inference/mlx_inference.py @@ -1189,7 +1189,8 @@ class MLXInferenceBackend: **gen_kwargs, ) - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): + # caller_cancel_event: signature parity with the orchestrator; unused here. import mlx.core as mx import gc diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py index 616384386d..4699148a08 100644 --- a/studio/backend/core/inference/orchestrator.py +++ b/studio/backend/core/inference/orchestrator.py @@ -104,6 +104,14 @@ class InferenceOrchestrator: # so a generate queued behind the cancelled one is skipped, not run. self._drain_event: Any = None self._gen_lock = threading.Lock() # Serializes generation + # Cancel event of the request holding _gen_lock: lets a Stop tell whether it owns the + # running generation or is queued behind it (the worker's event is shared). + self._active_cancel_events: list = [] + self._executing_cancel_events: list = [] + self._active_cancel_lock = threading.Lock() + # Held across claim + _send_cmd so claim order matches the subprocess dequeue order, + # which _owns_worker relies on. + self._send_order_lock = threading.Lock() # Set during a switch so a generation winning the _gen_lock handoff bails # instead of starting on the outgoing model. self._unload_pending = False @@ -112,6 +120,13 @@ class InferenceOrchestrator: # bypass _gen_lock, send commands directly, read from per-request # mailboxes routed by a dispatcher thread on request_id. self._mailboxes: dict[str, queue.Queue] = {} + # request_id -> cancel event, so the dispatcher can move worker ownership as it routes. + # Consumers read their mailbox whenever they get to it, so only the dispatcher sees + # responses in the order the worker produced them. + self._request_cancel_events: dict[str, object] = {} + # Mailboxes for the _gen_lock generations. Kept apart from _mailboxes because that map + # means "compare requests are in flight" to the unload and distributed paths. + self._direct_mailboxes: dict[str, queue.Queue] = {} self._mailbox_lock = threading.Lock() self._dispatcher_thread: Optional[threading.Thread] = None self._dispatcher_stop = threading.Event() @@ -321,9 +336,27 @@ class InferenceOrchestrator: self._resp_queue = None self._cancel_event = None self._drain_event = None + self._reset_worker_scoped_state() logger.info("Inference subprocess shut down") return True + def _reset_worker_scoped_state(self) -> None: + """Drop bookkeeping that only means anything for the worker that just died. + + Ownership is scoped by cancel-event identity alone, so a consumer still blocked + on its mailbox when the process was replaced stayed recorded as the executor. A + generation on the fresh worker then failed _owns_worker and could not be stopped. + Mailboxes go too: nothing will ever route to them, and a stale one reads as + compare activity to the unload path. + """ + with self._active_cancel_lock: + self._active_cancel_events.clear() + self._executing_cancel_events.clear() + with self._mailbox_lock: + self._mailboxes.clear() + self._direct_mailboxes.clear() + self._request_cancel_events.clear() + def _cleanup(self): """atexit handler.""" self._shutdown_subprocess(timeout = 5.0) @@ -463,6 +496,74 @@ class InferenceOrchestrator: except (EOFError, OSError, ValueError): return events + def _direct_reader(self, request_id: str): + """Response reader for a _gen_lock generation, safe once compare exists. + + The dispatcher and this reader would otherwise both consume _resp_queue. A + dispatcher started mid-stream took our responses and dropped them as + unaddressed (truncating or hanging the chat), and this reader, already blocked + on the queue, could take a compare request's response before that dispatcher + saw it. Registering a mailbox fixes the first; handing foreign responses to + their own mailbox fixes the second. + + Returns (read_one, drain, release). + """ + mailbox: queue.Queue = queue.Queue() + with self._mailbox_lock: + self._direct_mailboxes[request_id] = mailbox + + def read_one(timeout: float = 1.0): + try: + return mailbox.get_nowait() + except queue.Empty: + pass + thread = self._dispatcher_thread + if thread is not None and thread.is_alive(): + # It owns the queue now, and it routes to us. + try: + return mailbox.get(timeout = timeout) + except queue.Empty: + return None + resp = self._read_resp(timeout = timeout) + if resp is None: + return None + rid = resp.get("request_id") + if rid and rid != request_id: + with self._mailbox_lock: + other = self._mailboxes.get(rid) or self._direct_mailboxes.get(rid) + owner = self._request_cancel_events.get(rid) + if other is not None: + # We beat the dispatcher to this response, so make its ownership move here + # too. The compare consumer opts out of marking, so nothing else promotes + # or retires that request: skipping it left this one recorded as the + # executor, ignoring its Stop and letting a late reset cancel it. + if owner is not None: + if resp.get("type", "") in ("gen_done", "gen_error"): + self._release_worker(owner) + else: + self._mark_worker_started(owner) + other.put(resp) + return None + return resp + + def drain(timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + resp = read_one(timeout = min(0.5, deadline - time.monotonic())) + if resp is None: + if not self._ensure_subprocess_alive(): + return + continue + if resp.get("type", "") in ("gen_done", "gen_error"): + return + logger.warning("Timed out waiting for gen_done after cancel") + + def release() -> None: + with self._mailbox_lock: + self._direct_mailboxes.pop(request_id, None) + + return read_one, drain, release + def _drain_until_gen_done(self, timeout: float = 5.0) -> None: """Consume resp_queue events until gen_done/gen_error, discarding them. @@ -542,6 +643,7 @@ class InferenceOrchestrator: cancel_event = None, stats_holder: Optional[dict] = None, read_timeout: float = 30.0, + mark_started: bool = True, ) -> Generator[str, None, None]: """Yield tokens from a response stream until gen_done/gen_error. @@ -578,6 +680,11 @@ class InferenceOrchestrator: rtype = resp.get("type", "") if rtype == "status": continue + # The worker is answering THIS request, so it is the one executing: only now may its + # cancel event speak for the shared worker one. The dispatched path opts out: its + # dispatcher already did this in worker order, which a mailbox read can lag behind. + if mark_started: + self._mark_worker_started(cancel_event) # Subprocess-level error (no request_id); request-scoped failures # arrive as gen_error below. if rtype == "error" and not resp.get("request_id"): @@ -587,7 +694,13 @@ class InferenceOrchestrator: if rtype == "token": # Cancel from route (e.g. SSE connection closed). if cancel_event is not None and cancel_event.is_set(): - self._cancel_generation() + # Same rule as reset_generation_state: the shared worker event may only be set by + # the generation the worker is running. A dispatched request can still be draining + # stale mailbox tokens after the dispatcher retired it, and signalling from here + # would end the next one instead. Tearing this stream down is always safe, so the + # local drain happens either way. + if self._owns_worker(cancel_event): + self._cancel_generation() drain_on_cancel() return yield resp.get("text", "") @@ -681,8 +794,17 @@ class InferenceOrchestrator: # Route to mailbox if a matching request_id exists if rid: with self._mailbox_lock: - mbox = self._mailboxes.get(rid) + mbox = self._mailboxes.get(rid) or self._direct_mailboxes.get(rid) + owner = self._request_cancel_events.get(rid) if mbox is not None: + # Worker order, not consumer order: retire a request the moment its last response + # is routed. Waiting for the consumer's finally left it owning the worker after + # the worker moved on, so a late Stop for it cancelled whichever request started next. + if owner is not None: + if rtype in ("gen_done", "gen_error"): + self._release_worker(owner) + else: + self._mark_worker_started(owner) mbox.put(resp) continue @@ -798,6 +920,8 @@ class InferenceOrchestrator: ) if not unloading: self._mailboxes[request_id] = mailbox + if cancel_event is not None: + self._request_cancel_events[request_id] = cancel_event # When bailing without a mailbox, note whether any OTHER compare request still # routes through the dispatcher; if none and this call started it, stop it below. orphaned_dispatcher = unloading and not dispatcher_preexisting and not self._mailboxes @@ -813,11 +937,19 @@ class InferenceOrchestrator: yield GenStreamError("Error: model is being unloaded", public = True) return + # Claim before sending, like the locked path: dispatched runs are concurrent by design, + # so without this a Stop on one saw no owner and reset the worker, ending its siblings. + # Claim and enqueue under one lock, or two dispatcher threads interleave and claim order + # stops matching the subprocess's command order, which _owns_worker reads. try: - self._send_cmd(cmd) + with self._send_order_lock: + self._claim_worker(cancel_event) + self._send_cmd(cmd) except RuntimeError as exc: + self._release_worker(cancel_event) with self._mailbox_lock: self._mailboxes.pop(request_id, None) + self._request_cancel_events.pop(request_id, None) yield GenStreamError(f"Error: {exc}") return @@ -836,10 +968,15 @@ class InferenceOrchestrator: cancel_event = cancel_event, stats_holder = stats_holder, read_timeout = _DISPATCH_READ_TIMEOUT, + mark_started = False, ) finally: + # Normally already retired by the dispatcher at gen_done; this covers streams that + # end without one (cancel, disconnect, a dead subprocess). + self._release_worker(cancel_event) with self._mailbox_lock: self._mailboxes.pop(request_id, None) + self._request_cancel_events.pop(request_id, None) def _drain_mailbox( self, @@ -1578,6 +1715,11 @@ class InferenceOrchestrator: # Won the lock handoff during a switch; don't start on the outgoing model. yield GenStreamError("Error: model is being unloaded", public = True) return + if cancel_event is not None and cancel_event.is_set(): + # Stopped while queued on the lock. Sending anyway occupied the worker with a + # run the user ended: the cancel is only seen on a token, so a long prefill + # (or a generation that reaches gen_done without one) held up its siblings. + return request_id = str(uuid.uuid4()) image_b64 = self._pil_to_base64(image) if image is not None else None cmd = self._build_generate_cmd( @@ -1599,22 +1741,95 @@ class InferenceOrchestrator: preserve_thinking = preserve_thinking, ) + # Claim the worker BEFORE sending, so a Stop on some OTHER chat -- still queued on the + # lock above, having generated nothing -- cannot reset the generation this is starting. + # Claiming after the send left the command running unclaimed. Released in the finally. + # Own mailbox: a compare request can start the dispatcher while this is streaming, + # and it would otherwise consume our responses and drop them. + read_one, drain, release_mailbox = self._direct_reader(request_id) try: - self._send_cmd(cmd) - except RuntimeError as exc: - yield GenStreamError(f"Error: {exc}") - return + try: + with self._send_order_lock: + self._claim_worker(cancel_event) + self._send_cmd(cmd) + except RuntimeError as exc: + yield GenStreamError(f"Error: {exc}") + return - yield from self._consume_token_stream( - self._read_resp, - lambda: self._drain_until_gen_done(timeout = 5.0), - crash_context = "generation", - cancel_event = cancel_event, - stats_holder = stats_holder, - ) + yield from self._consume_token_stream( + read_one, + lambda: drain(timeout = 5.0), + crash_context = "generation", + cancel_event = cancel_event, + stats_holder = stats_holder, + ) + finally: + self._release_worker(cancel_event) + release_mailbox() - def reset_generation_state(self): - """Cancel any ongoing generation and reset state.""" + def _claim_worker(self, cancel_event) -> None: + """Record this request as one the worker will run. + + Admission only. The subprocess executes generations one at a time, so a + dispatched request sitting behind another in the command queue is claimed + but not executing, and must not be able to signal the shared cancel event + (that would end whichever request IS executing). _mark_worker_started + promotes it once the worker answers it. + """ + with self._active_cancel_lock: + self._active_cancel_events.append(cancel_event) + + def _mark_worker_started(self, cancel_event) -> None: + """Promote a claimed request to executing, on its first worker response. + + Sole executor: the subprocess runs one generation at a time, so answering + this one means it has left the previous one behind. + """ + if cancel_event is None: + return + with self._active_cancel_lock: + if self._executing_cancel_events[:1] != [cancel_event]: + self._executing_cancel_events[:] = [cancel_event] + + def _release_worker(self, cancel_event) -> None: + with self._active_cancel_lock: + for bucket in (self._active_cancel_events, self._executing_cancel_events): + try: + bucket.remove(cancel_event) + except ValueError: + pass + + def _owns_worker(self, cancel_event) -> bool: + """Whether a reset from this request may signal the shared cancel event. + + True when it is one of the EXECUTING generations, and when nothing is in + flight at all: an error path that resets before anything started has no + one else to interrupt, so it must not become a silent no-op. Claimed but + queued does not count, or a Stop on a queued request would end the + running one, including during the prefill before any response arrives. + """ + with self._active_cancel_lock: + if not self._active_cancel_events: + # Nothing in flight at all, so there is no one to protect. + return True + if self._executing_cancel_events: + return any(ev is cancel_event for ev in self._executing_cancel_events) + # Claimed but nothing has answered yet (A is in prefill). The worker takes commands + # in order, so the oldest claim is the executor; anyone else here is queued behind it. + return self._active_cancel_events[0] is cancel_event + + def reset_generation_state(self, caller_cancel_event = None): + """Cancel any ongoing generation and reset state. + + ``caller_cancel_event`` scopes the reset to one request. The worker has a + single cancel event and generation is serialized on _gen_lock, so a chat + that is still queued has no generation of its own to reset: calling this + from its Stop handler would kill whichever chat currently holds the lock. + Pass the request's own event and the reset is dropped unless that request + is the one running. Omit it for genuinely global resets (unload, switch). + """ + if caller_cancel_event is not None and not self._owns_worker(caller_cancel_event): + return self._cancel_generation() if not self._ensure_subprocess_alive(): return @@ -1673,35 +1888,40 @@ class InferenceOrchestrator: if use_adapter is not None: cmd["use_adapter"] = use_adapter - self._send_cmd(cmd) + # Same shared-queue hazard as _generate_inner: see _direct_reader. + read_one, _drain, release_mailbox = self._direct_reader(request_id) + try: + self._send_cmd(cmd) - deadline = time.monotonic() + 120.0 - while time.monotonic() < deadline: - remaining = max(0.1, deadline - time.monotonic()) - resp = self._read_resp(timeout = min(remaining, 1.0)) + deadline = time.monotonic() + 120.0 + while time.monotonic() < deadline: + remaining = max(0.1, deadline - time.monotonic()) + resp = read_one(timeout = min(remaining, 1.0)) - if resp is None: - if not self._ensure_subprocess_alive(): - raise RuntimeError(self._subprocess_crash_message("audio generation")) - continue + if resp is None: + if not self._ensure_subprocess_alive(): + raise RuntimeError(self._subprocess_crash_message("audio generation")) + continue - rtype = resp.get("type", "") + rtype = resp.get("type", "") - if rtype == "audio_done": - wav_bytes = base64.b64decode(resp["wav_base64"]) - sample_rate = resp["sample_rate"] - return wav_bytes, sample_rate + if rtype == "audio_done": + wav_bytes = base64.b64decode(resp["wav_base64"]) + sample_rate = resp["sample_rate"] + return wav_bytes, sample_rate - if rtype == "audio_error": - raise RuntimeError(resp.get("error", "Audio generation failed")) + if rtype == "audio_error": + raise RuntimeError(resp.get("error", "Audio generation failed")) - if rtype == "error": - raise RuntimeError(resp.get("error", "Unknown error")) + if rtype == "error": + raise RuntimeError(resp.get("error", "Unknown error")) - if rtype == "status": - continue + if rtype == "status": + continue - raise RuntimeError("Timeout waiting for audio generation (120s)") + raise RuntimeError("Timeout waiting for audio generation (120s)") + finally: + release_mailbox() def generate_whisper_response( self, @@ -1775,6 +1995,9 @@ class InferenceOrchestrator: # Won the lock handoff during a switch; don't start on the outgoing model. yield GenStreamError("Error: model is being unloaded", public = True) return + if cancel_event is not None and cancel_event.is_set(): + # Stopped while queued on the lock, same as _generate_inner. + return request_id = str(uuid.uuid4()) # numpy array -> list for mp.Queue serialization @@ -1797,18 +2020,28 @@ class InferenceOrchestrator: "repetition_penalty": repetition_penalty, } + # Same shared-queue hazard as _generate_inner: see _direct_reader. + read_one, drain, release_mailbox = self._direct_reader(request_id) try: - self._send_cmd(cmd) - except RuntimeError as exc: - yield GenStreamError(f"Error: {exc}") - return + try: + # Claim under the send lock, like _generate_inner: unclaimed, a compare request queued + # behind this looked like the oldest owner, so stopping it killed this one. + with self._send_order_lock: + self._claim_worker(cancel_event) + self._send_cmd(cmd) + except RuntimeError as exc: + yield GenStreamError(f"Error: {exc}") + return - yield from self._consume_token_stream( - self._read_resp, - lambda: self._drain_until_gen_done(timeout = 5.0), - crash_context = "audio input generation", - cancel_event = cancel_event, - ) + yield from self._consume_token_stream( + read_one, + lambda: drain(timeout = 5.0), + crash_context = "audio input generation", + cancel_event = cancel_event, + ) + finally: + self._release_worker(cancel_event) + release_mailbox() # ------------------------------------------------------------------ # Local helpers (no subprocess needed) diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py index 9345ce3f87..b593bc119b 100644 --- a/studio/backend/core/inference/safetensors_agentic.py +++ b/studio/backend/core/inference/safetensors_agentic.py @@ -59,6 +59,7 @@ from core.tool_healing import ( from core.inference.tool_loop_controller import ( ToolLoopController, append_deferred_nudges, + awaiting_approval_status, coerce_tool_arguments, status_for_tool, tool_event_provenance, @@ -1209,18 +1210,30 @@ def run_safetensors_tool_loop( start_event["awaiting_confirmation"] = needs_confirm try: - yield {"type": "status", "text": decision.status_text} + # A gated call has not started: say waiting, not "Running" (GGUF parity). + yield { + "type": "status", + "text": ( + awaiting_approval_status(decision.tool_name) + if needs_confirm + else decision.status_text + ), + } yield start_event - if ( - decision_slot is not None - and wait_tool_decision( + _decision = ( + wait_tool_decision( decision_slot, approval_id, cancel_event = cancel_event, ) - == "deny" - ): + if decision_slot is not None + else None + ) + if _decision is not None and _decision != "deny": + # Approved: now it really is running. + yield {"type": "status", "text": decision.status_text} + if _decision == "deny": decision_slot = None if provisional_match: provisional_resolved = True diff --git a/studio/backend/core/inference/tool_loop_controller.py b/studio/backend/core/inference/tool_loop_controller.py index 361f4b20e3..feedae5874 100644 --- a/studio/backend/core/inference/tool_loop_controller.py +++ b/studio/backend/core/inference/tool_loop_controller.py @@ -238,6 +238,19 @@ def status_for_tool(tool_name: str, arguments: Mapping[str, Any]) -> str: return f"Calling: {tool_name}" +def awaiting_approval_status(tool_name: str) -> str: + """Status text for a call parked on the approval prompt. + + It has not started, so reporting "Running ..." with a climbing timer reads + as a hang. + """ + if tool_name == "python": + return "Waiting for approval: Python" + if tool_name == "terminal": + return "Waiting for approval: command" + return f"Waiting for approval: {tool_name}" + + def is_tool_error(result: str) -> bool: return isinstance(result, str) and result.lstrip().startswith(TOOL_ERROR_PREFIXES) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index bd5322819e..0c6e2292bc 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -3105,6 +3105,22 @@ def is_always_safe_tool(name: str) -> bool: return name in _ALWAYS_SAFE_TOOLS +# Tools whose provisional card is only a text preview of the arguments, so it can stream +# while awaiting approval. +_TEXT_PREVIEW_TOOLS = frozenset({"python", "terminal"}) + + +def has_text_only_provisional_card(name: str) -> bool: + """True when streaming this tool's arguments before approval shows only text. + + A large code payload takes a minute or more to write, and suppressing the + card until the call completes leaves the chat blank the whole time. Nothing + runs before the decision either way, and you have to read the code to make + it. + """ + return name in _TEXT_PREVIEW_TOOLS + + def is_potentially_unsafe_tool_call(name: str, arguments: dict) -> bool: """Whether a tool call must still pause for approval in auto mode. diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index fe59bc3e78..e66adb789e 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -191,12 +191,26 @@ class LoadRequest(BaseModel): "auth, UI/server mode) are rejected. Ignored for non-GGUF models." ), ) + force_cancel_active: bool = Field( + False, + description = ( + "Stop chats still generating instead of refusing with 409. A load " + "replaces the llama-server every open conversation decodes on." + ), + ) class UnloadRequest(BaseModel): """Request to unload a model""" model_path: str = Field(..., description = "Model identifier to unload") + force_cancel_active: bool = Field( + False, + description = ( + "Stop chats still generating instead of refusing with 409. An " + "unload takes away the llama-server they are decoding on." + ), + ) class TranscribeRequest(BaseModel): @@ -350,6 +364,14 @@ class InstallLatestTransformersRequest(BaseModel): description = "Exact transformers version to install; must match the current " "latest PyPI release reported by /validate.", ) + force_cancel_active: bool = Field( + False, + description = ( + "Stop chats still generating instead of refusing with 409. The install " + "is a step of the model swap that raised the same prompt, so a client " + "that already got consent for that swap can carry it through here." + ), + ) class InstallLatestTransformersResponse(BaseModel): diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 9843dc6378..97149f7a17 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1796,6 +1796,7 @@ from core.inference.anthropic_compat import ( AnthropicPassthroughEmitter, ) from auth.authentication import get_current_subject +from state import active_generations from state.tool_approvals import resolve_tool_decision from core.inference.key_exchange import decrypt_api_key @@ -2246,11 +2247,38 @@ def _prune_pending(now: float) -> None: class _TrackedCancel: - """Register cancel_event in _CANCEL_REGISTRY for the block's duration.""" + """Register cancel_event in _CANCEL_REGISTRY for the block's duration. - def __init__(self, event: threading.Event, *keys): + Also records the run in state.active_generations so /load and /unload can + see which chats a reload would interrupt. Both registries share this event, + so either one cancels down the same per-request path. + """ + + def __init__( + self, + event: threading.Event, + *keys, + thread_id = None, + model = None, + kind = "chat", + ): self.event = event self.keys = tuple(k for k in keys if k) + # kind reaches the swap prompt: embeddings and raw completions have no conversation, so + # naming them chats would offer to stop something the user never started from a thread. + self._active = active_generations.ActiveGeneration( + event, thread_id = thread_id, model = model, kind = kind + ) + + @classmethod + def for_payload(cls, event: threading.Event, payload, *keys): + """Track the run against the conversation its request names.""" + return cls( + event, + *keys, + thread_id = getattr(payload, "thread_id", None), + model = getattr(payload, "model", None), + ) def __enter__(self): # Register + consume-pending in one critical section to close the @@ -2264,6 +2292,7 @@ class _TrackedCancel: for k in self.keys: if k and _PENDING_CANCELS.pop(k, None) is not None: should_cancel = True + self._active.__enter__() if should_cancel: self.event.set() return self.event @@ -2277,6 +2306,7 @@ class _TrackedCancel: bucket.discard(self.event) if not bucket: _CANCEL_REGISTRY.pop(k, None) + self._active.__exit__(*exc) return False @@ -3502,15 +3532,38 @@ def _switch_waiter_count() -> int: return sum(max(0, count) for count in _auto_switch_waiters.values()) -async def _wait_for_model_switch_idle(*, current_request_counted: bool) -> None: +async def _wait_for_model_switch_idle( + *, + current_request_counted: bool, + cancel_pending: bool = False, + timeout_s: Optional[float] = None, +) -> None: """Wait until a model replacement cannot interrupt active inference. The caller holds ``inference_lifecycle_gate``, which prevents new inference from starting while existing requests drain. Auto-switch requests that have resolved their targets are scheduler waiters, not active generations, so exclude them to avoid a queue deadlock. + + ``cancel_pending`` is set by a forced swap that has NOT cancelled yet: the + registered generations are the ones it is about to stop, so waiting on them + would wait out exactly what the force exists to end. Excluding them lets the + drain finish ahead of the cancel, which keeps every check that can still + reject the swap in front of the destructive step. Recomputed each poll (not + snapshotted) so a generation that ends on its own stops being discounted and + the remaining, non-cancellable requests are still waited out. + + ``timeout_s`` bounds the wait and returns rather than raising. Only the + post-cancel drains pass it: what they wait on may never observe its cancel + (TTS on the subprocess backend has no observer), and they hold the lifecycle + gate, so an unbounded wait pins every load and unload behind one + uninterruptible generation. Expiring there just proceeds, which is what they + do anyway once drained. Pre-cancel drains stay unbounded -- the swap can + still be refused, so they must not shorten the protection they provide. """ from core.inference.llama_keepwarm import other_inference_request_count + + deadline = None if timeout_s is None else time.monotonic() + timeout_s while True: queued_switches = _switch_waiter_count() if current_request_counted and queued_switches > 0: @@ -3519,8 +3572,19 @@ async def _wait_for_model_switch_idle(*, current_request_counted: bool) -> None: current_request_counted = current_request_counted, include_pending = False, ) + if cancel_pending: + active_others -= min(active_others, active_generations.count()) if active_others <= queued_switches: return + if deadline is not None and time.monotonic() >= deadline: + logger.warning( + "model_switch_drain_timed_out", + extra = { + "event": "inference.switch_drain_timeout", + "remaining": active_others - queued_switches, + }, + ) + return await asyncio.sleep(0.02) @@ -4796,6 +4860,214 @@ def _raise_if_sidecar_swap_in_progress() -> None: ) +def _raise_or_cancel_active_generations( + *, + force: bool, + action: str, + cancel: bool = True, +) -> int: + """Gate a model swap on the chats currently generating. + + Every open conversation decodes on the single llama-server this route is + about to replace, so refuse with 409 and name them. force_cancel_active + instead stops them through the same events an explicit Stop uses. Returns + how many were cancelled. The frontend guard is bypassable from a second tab + or curl; this one is not. + + ``cancel = False`` runs the refusal half only. /load calls it that way once + up front, so a non-forced swap still fails fast, and again with cancel just + before teardown: cancelling is destructive and unrecoverable, so it must not + run ahead of preflight checks that can still reject the load (see + _load_model_impl). + """ + if not active_generations.count(): + return 0 + if not force: + thread_ids = active_generations.active_thread_ids() + running = active_generations.count() + raise HTTPException( + status_code = 409, + detail = { + "error": "active_generations", + "message": ( + f"{action} would stop {running} chat" + f"{'s' if running != 1 else ''} that " + f"{'are' if running != 1 else 'is'} still generating. " + "Stop them first, or retry with force_cancel_active." + ), + "running": running, + "thread_ids": thread_ids, + }, + ) + if not cancel: + # Refusal-only pass: the caller cancels later, once nothing can still reject the load. + return 0 + cancelled = active_generations.cancel_all() + if cancelled: + logger.info( + "model_swap_cancelled_active_generations", + extra = {"event": "inference.reload_cancelled_generations", "count": cancelled}, + ) + return cancelled + + +_POST_CANCEL_DRAIN_TIMEOUT_S = 5.0 + + +async def _cancel_and_drain_for_sidecar_swap(timeout_s: Optional[float] = None) -> None: + """Clear the way for a confirmed sidecar swap, then stop the chats it interrupts. + + The installer gates on the middleware's in-flight count, not on + active_generations, so it also sees requests the cancel cannot stop. Drain + those FIRST, discounting the registered chats (they are what the cancel is + for, so waiting on them would wait out the point of the force). Only then + cancel, and let the survivors unwind. Cancelling first meant an unrelated + counted request -- a /v1/messages/count_tokens, say -- was still there for + the caller's recheck, which then refused an install that had already stopped + every chat for nothing. + + Bounded on both halves: the requests being waited on may never observe a + cancel, and this holds the lifecycle gate and the sidecar reservation inside + ``asyncio.shield``, so an unbounded wait would wedge the process. Expiring in + the first half returns without cancelling, so the caller's recheck refuses + with the chats untouched. + """ + from core.inference.llama_keepwarm import other_inference_request_count + + budget = _POST_CANCEL_DRAIN_TIMEOUT_S if timeout_s is None else timeout_s + + async def _drain(deadline: float, *, discount_registered: bool) -> bool: + while True: + counted = other_inference_request_count( + current_request_counted = False, include_pending = False + ) + if discount_registered: + counted -= min(counted, active_generations.count()) + if counted <= 0: + return True + if time.monotonic() >= deadline: + return False + await asyncio.sleep(0.02) + + # Weighted, not halved, so the total wait under the gate is unchanged. The first drain only + # asks whether unrelated inference is in flight; cutting the second short refused installs + # whose chats had already been stopped for nothing. + if not await _drain(time.monotonic() + budget / 5, discount_registered = True): + return + _raise_or_cancel_active_generations(force = True, action = "Installing a new transformers version") + await _drain(time.monotonic() + budget * 4 / 5, discount_registered = False) + + +async def _drain_and_recancel_before_teardown(*, force: bool, action: str) -> None: + """Wait out inference the registry cannot see, then stop anything new. + + A request that passed the keep-warm middleware but has not reached its + ``_TrackedCancel`` yet is counted in-flight and absent from the registry, so + cancelling on the registry alone lets a teardown land on an already-admitted + request. Drain on the middleware count instead, which covers both the runs + just cancelled and the ones still in that window, then cancel again for + anything that registered while waiting. + + Bounded and non-raising: an unload is a deliberate user action, so the worst + case stays what it is today rather than becoming a refusal. + """ + await _wait_for_model_switch_idle( + current_request_counted = False, + timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S, + ) + if force: + _raise_or_cancel_active_generations(force = True, action = action) + + +_UNRESOLVED_BACKEND_STATE = object() + + +def _unload_evicts_standard_backend(backend, model_path: str) -> bool: + """Whether ``backend.unload_model(model_path)`` will really evict something. + + The standard backend refuses to unload a name it never loaded ("don't unload + a stale model") and returns success, so /unload for a model another tab has + already replaced is a no-op. That must not count as a teardown: cancelling + the running chats for it would end them and leave the resident model up. + + Mirrors the backend's own guard (case-insensitive on the active name, since + the load path canonicalizes casing). A backend that exposes neither field is + reported as a real unload, which keeps the previous behaviour. + """ + active = getattr(backend, "active_model_name", _UNRESOLVED_BACKEND_STATE) + loaded = getattr(backend, "models", _UNRESOLVED_BACKEND_STATE) + if active is _UNRESOLVED_BACKEND_STATE and loaded is _UNRESOLVED_BACKEND_STATE: + return True + if isinstance(active, str) and active and active.lower() == (model_path or "").lower(): + return True + return isinstance(loaded, dict) and model_path in loaded + + +def _unload_may_evict(model_path: str) -> bool: + """Whether POST /unload for ``model_path`` can still tear something down. + + The refusal passes gate on this. A request naming a model another tab has + already replaced reaches none of the teardown branches and returns the + documented idempotent no-op (see _unload_evicts_standard_backend), so + refusing it counts a teardown that cannot happen and leaves a stale tab + unable to clear its selection. Each disjunct mirrors one teardown branch, so + True means "some branch may fire", never "this unload succeeds". + + Attribute reads only, no lifecycle gate, so the pre-gate pass still fails + fast on a swap that would really stop chats. A stale answer is safe in both + directions: the gated pass re-runs this under the gate, and every branch + re-runs the refusal at its own point of no return, so a False here can never + let a teardown through unrefused. + """ + backend = get_inference_backend() + loading = getattr(backend, "get_loading_model", lambda: None)() + if ( + loading is not None + and hasattr(backend, "cancel_load") + and (model_path == loading or model_path.lower() == loading.lower()) + ): + return True + llama_backend = get_llama_cpp_backend() + if llama_backend.is_active and ( + llama_backend.model_identifier == model_path + or is_registered_native_path_label(llama_backend.model_identifier, model_path) + # Up but not serving is mid-load, evicted whatever model was named. + or not llama_backend.is_loaded + ): + return True + return _unload_evicts_standard_backend(backend, model_path) + + +@studio_router.get("/active-generations") +async def get_active_generations( + fastapi_request: Request, current_subject: str = Depends(get_current_subject) +): + """Conversations currently generating, plus how many can decode at once. + + Lets a model swap name the chats it would interrupt, including runs this tab + cannot see (another tab, or a reload behind a proxy). parallel_slots is the + slot count actually in use, which the VRAM fit may have cut below the + requested --parallel; chats beyond it queue rather than fail. + """ + entries = active_generations.snapshot() + # A tracker's model can be a native local path (the legacy stream records active_model_name + # verbatim); redact here, the one place that serialises it. + for _entry in entries: + if isinstance(_entry.get("model"), str): + _entry["model"] = redact_native_paths(_entry["model"]) + slots = 1 + try: + slots = _openai_llama_admission_capacity(fastapi_request, get_llama_cpp_backend()) + except Exception: + slots = int(getattr(fastapi_request.app.state, "llama_parallel_slots", 1) or 1) + return { + "active": entries, + "count": len(entries), + "thread_ids": active_generations.active_thread_ids(), + "parallel_slots": max(1, int(slots)), + } + + @router.post("/load", response_model = LoadResponse) async def load_model( request: LoadRequest, @@ -4823,7 +5095,18 @@ async def load_model( # holds this gate. async with inference_lifecycle_gate(): _raise_if_sidecar_swap_in_progress() - return await _load_model_impl(request, fastapi_request, current_subject) + # The active-generation gate runs inside _load_model_impl, once it knows this is a real + # reload, and still under the lifecycle gate so the check stays atomic with the teardown. + return await _load_model_impl( + request, + fastapi_request, + current_subject, + on_reload_confirmed = lambda *, cancel: _raise_or_cancel_active_generations( + force = request.force_cancel_active, + action = "Loading a model", + cancel = cancel, + ), + ) async def _load_model_impl( @@ -4832,6 +5115,7 @@ async def _load_model_impl( current_subject: str, *, current_request_counted: bool = False, + on_reload_confirmed = None, ): from core.inference.llama_cpp import LlamaServerNotFoundError @@ -5041,6 +5325,19 @@ async def _load_model_impl( chat_template = _chat_template, ) + # Past every already_loaded fast return, so this really will replace the running model: gate + # it on the chats that would stop. Refusal only, so a non-forced swap fails fast; the checks + # between here and the teardown (identifier, GPU, training guard, downloads) can still + # reject the load, and cancelling now would stop every chat for a model that never loads. + # Auto-switch passes no hook and keeps its current behaviour. + if on_reload_confirmed is not None: + on_reload_confirmed(cancel = False) + + # Destructive cancel still owed at the teardown below, so it can be deferred past every + # remaining check; the drains key off this. Only a forced swap cancels: unforced already + # 409'd above, auto-switch has no hook. + cancel_pending = on_reload_confirmed is not None and bool(request.force_cancel_active) + # is_lora auto-detected from adapter_config.json on disk/HF. # DNS-probe wrap so offline loads skip 30-60s of soft-failed network # checks before the worker starts. @@ -5154,13 +5451,33 @@ async def _load_model_impl( ), ) - # Keep the resident model alive until every active generation finishes; - # the caller's lifecycle gate blocks new starts. - await _wait_for_model_switch_idle(current_request_counted = current_request_counted) - # A sidecar install can reserve the gate while inference drains, after the - # route-level checks above, so recheck before replacing either backend. + # Fast path only: a swap can still be reserved during the drain. _raise_if_sidecar_swap_in_progress() + # Drain active generations first (the lifecycle gate blocks new starts); a forced swap + # excludes the ones it is about to cancel rather than waiting them out. + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + cancel_pending = cancel_pending, + ) + # Decisive recheck, and the last thing that can reject this load, so it runs BEFORE the + # cancel: rejecting after would stop every chat for nothing. + _raise_if_sidecar_swap_in_progress() + + # Point of no return for the GGUF path: nothing left can reject this load, so stop the + # chats the swap interrupts (or refuse, if the caller never opted in). + if on_reload_confirmed is not None: + on_reload_confirmed(cancel = True) + + # Let the cancelled generations unwind before the teardown; no check follows, so this cannot + # strand a cancelled chat behind a 409. Bounded: TTS observes no cancel event, so an + # unbounded wait would hold the gate for a whole audio run. + if cancel_pending: + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S, + ) + # Unload any active Unsloth model only after every hub conflict check. if unsloth_backend.active_model_name: logger.info( @@ -5376,10 +5693,27 @@ async def _load_model_impl( # ── Standard path: load via Unsloth/transformers ────────── backend = get_inference_backend() - # Unload any active GGUF model first - llama_backend = get_llama_cpp_backend() - await _wait_for_model_switch_idle(current_request_counted = current_request_counted) + # Same sidecar rejection as GGUF: fast path ahead of the drain, rechecked after. _raise_if_sidecar_swap_in_progress() + + llama_backend = get_llama_cpp_backend() + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + cancel_pending = cancel_pending, + ) + _raise_if_sidecar_swap_in_progress() + + # Point of no return for the Unsloth path: cancel only once nothing can still reject the load. + if on_reload_confirmed is not None: + on_reload_confirmed(cancel = True) + + # Let the cancelled generations unwind before the teardown; no check follows. Bounded like GGUF. + if cancel_pending: + await _wait_for_model_switch_idle( + current_request_counted = current_request_counted, + timeout_s = _POST_CANCEL_DRAIN_TIMEOUT_S, + ) + # Unload any active GGUF model first if llama_backend.is_loaded: logger.info("Unloading GGUF model before loading Unsloth model") llama_backend.unload_model() @@ -5978,7 +6312,13 @@ async def install_latest_transformers_route( other_inference_request_count, ) - if other_inference_request_count(current_request_counted = False, include_pending = False) > 0: + # A confirmed swap skips only this fast path; the recheck under the gate still has to pass, + # so the guard is unchanged for anyone who did not confirm. + if ( + not request.force_cancel_active + and other_inference_request_count(current_request_counted = False, include_pending = False) + > 0 + ): raise HTTPException( status_code = 409, detail = ( @@ -6072,9 +6412,16 @@ async def install_latest_transformers_route( "Retry the install." ), ) + # Carry a confirmed swap's decision through: the user already accepted the "stop N + # chats" prompt, and refusing here would make that answer unactionable (Retry + # cannot succeed while the same chats run). Deliberately LAST, after every check + # that can still reject the install, so the cancel is spent only once nothing can + # turn this request away -- /load's rule. + if request.force_cancel_active: + await _cancel_and_drain_for_sidecar_swap() # Recheck under the gate: new streams bump their in-flight count while - # holding it, so once held nothing slips past (the pre-gate check is only - # a fast path and can be outlasted by a wait on a long /load). + # holding it, so once held nothing slips past. A forced install that could + # not drain in time lands here too, for the same 409 as without the flag. if ( other_inference_request_count( current_request_counted = False, include_pending = False @@ -6126,9 +6473,9 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge from core.inference.llama_keepwarm import inference_lifecycle_gate, note_model_unloaded try: # "Stop loading" (frontend cancelLoading -> /unload) must abort a still-loading - # model promptly. /load holds the lifecycle gate for the whole (multi-minute) load, - # so gating first would make the cancel wait it out. cancel_load only tears the - # loading subprocess down (no unload command), so it is safe off-gate. + # model promptly, and /load holds the lifecycle gate for the whole load. cancel_load only + # tears the loading subprocess down, so it is safe off-gate -- and ahead of the + # active-generation refusal below, which it can never need (see there). backend = get_inference_backend() loading = getattr(backend, "get_loading_model", lambda: None)() if ( @@ -6141,13 +6488,11 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge logger.info(f"Cancelled in-flight load: {request.model_path}") return UnloadResponse(status = "unloaded", model = request.model_path) - # Same "stop loading" fast path for a still-loading GGUF (llama-server spawned, - # health check not yet passed). A gated unload would wait out the multi-minute - # load; unload_model() sets the cancel_event load_model polls off its own lock and - # kills the child, sending no worker command, so it is safe off-gate like - # cancel_load. The gated GGUF branch below handles the already-loaded case. Gate on - # the loading model (identifier or native label): the single llama-server loads one - # GGUF at a time, so an unload for a different model must not cancel this load. + # Same "stop loading" fast path for a still-loading GGUF (spawned, health check not passed). + # unload_model() sets the cancel_event load_model polls and kills the child without a + # worker command, so it is safe off-gate like cancel_load; the gated branch below handles + # the already-loaded case. Gated on the loading model so an unload for a different model + # cannot cancel this load. llama_backend = get_llama_cpp_backend() if ( llama_backend.is_active @@ -6164,11 +6509,35 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge logger.info(f"Cancelled in-flight GGUF load: {request.model_path}") return UnloadResponse(status = "unloaded", model = request.model_path) + # Same gate as /load: refusal only, so a non-forced unload fails fast before queueing on the + # lifecycle gate. Skipped when no teardown branch can fire, or a request naming a model + # another tab already replaced would 409 on chats it cannot interrupt. + # + # BEHIND the two "stop loading" fast paths above: both cancel a load that has not replaced + # anything yet, so neither can interrupt a chat, and refusing them counted a teardown that + # cannot happen (unretryably -- the frontend's Cancel sends this unload unforced and drops + # the error). Any other name still falls through here. + if _unload_may_evict(request.model_path): + _raise_or_cancel_active_generations( + force = request.force_cancel_active, + action = "Unloading the model", + cancel = False, + ) + # Serialize with /load under the same lifecycle gate: the Unsloth unload now runs # off the event loop (asyncio.to_thread), so without this a concurrent /load could # swap in a fresh subprocess mid-unload and the unload command would land on the # new worker. The gate makes load and unload exclusive. async with inference_lifecycle_gate(): + # Rechecked under the gate, like /load: a chat can register while this one queues here (the + # middleware takes and releases the same gate). Still refusal only, and re-read rather + # than carried down, since a load may have finished meanwhile. + if _unload_may_evict(request.model_path): + _raise_or_cancel_active_generations( + force = request.force_cancel_active, + action = "Unloading the model", + cancel = False, + ) # Check if the GGUF backend has this model loaded or is loading it. llama_backend = get_llama_cpp_backend() if llama_backend.is_active and ( @@ -6181,8 +6550,18 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge # Read the identity before teardown clears it, so the row reads repo:QUANT. _unloaded = _llama_public_model_id(llama_backend, request.model_path) _unloaded_variant = getattr(llama_backend, "hf_variant", None) - # A manual unload is a deliberate user action: tear down now even if a - # request is mid-stream (only the automatic idle loop defers to it). + # Point of no return: this really does replace the running server, so stop the + # chats. A manual unload is a deliberate user action, so it cancels mid-stream + # requests rather than deferring to them the way the automatic idle loop does. + _raise_or_cancel_active_generations( + force = request.force_cancel_active, action = "Unloading the model" + ) + # Let what we just cancelled unwind first, like /load: tearing the server down under + # streams told to stop but not yet finished turned a clean end into a dropped + # connection. Bounded, since a manual unload is deliberate. + await _drain_and_recancel_before_teardown( + force = request.force_cancel_active, action = "Unloading the model" + ) llama_backend.unload_model() note_model_unloaded() api_monitor.record_lifecycle( @@ -6197,6 +6576,14 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge # a slow SSE stream paused between tokens still holds, so a sync call would block # the loop that drives the stream's next token and the lock release. backend = get_inference_backend() + if _unload_evicts_standard_backend(backend, request.model_path): + # Point of no return for the standard path, same rule as above. + _raise_or_cancel_active_generations( + force = request.force_cancel_active, action = "Unloading the model" + ) + await _drain_and_recancel_before_teardown( + force = request.force_cancel_active, action = "Unloading the model" + ) await asyncio.to_thread(backend.unload_model, request.model_path) note_model_unloaded() api_monitor.record_lifecycle( @@ -6207,6 +6594,9 @@ async def unload_model(request: UnloadRequest, current_subject: str = Depends(ge logger.info(f"Unloaded model: {request.model_path}") return UnloadResponse(status = "unloaded", model = request.model_path) + except HTTPException: + # Typed refusals (the gate's 409) must not be rewritten as a 500 below. + raise except Exception as e: logger.error(f"Error unloading model: {e}", exc_info = True) raise HTTPException(status_code = 500, detail = "Failed to unload model") @@ -6355,6 +6745,12 @@ async def generate_stream( disconnect_watcher = asyncio.create_task( _await_disconnect_then_cancel(fastapi_request, cancel_event) ) + # Registered inside the generator, under the finally that unregisters it, so a response whose + # body never starts leaves nothing behind. Unregistered, this run passes /unload's 409 gate + # (which runs no idle drain) and a forced swap has no event to signal. GenerateRequest + # carries no thread_id: counted, not nameable. + _tracker = _TrackedCancel(cancel_event, model = backend.active_model_name) + _tracker.__enter__() try: gen = backend.generate_chat_response( messages = request.messages, @@ -6375,7 +6771,7 @@ async def generate_stream( # Watcher set cancel_event between chunks. Reset here: closing # the generator does not signal a subprocess backend, so it would # keep decoding. The finally's reset is guarded, so no double-run. - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) break chunk = await asyncio.to_thread(next, gen, _DONE) if chunk is _DONE: @@ -6391,24 +6787,28 @@ async def generate_stream( except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) raise except Exception as e: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) logger.error(f"Error during generation: {e}", exc_info = True) yield f"data: {json.dumps({'error': _friendly_error(e)})}\n\n" yield "data: [DONE]\n\n" finally: - await _stop_local_disconnect_cancel_watcher(disconnect_watcher) - if not completed and not cancel_event.is_set(): - cancel_event.set() - backend.reset_generation_state() - if gen is not None: - try: - await asyncio.to_thread(gen.close) - except (RuntimeError, ValueError): - pass + # Nested so a teardown failure still unregisters; a phantom entry 409s swaps. + try: + await _stop_local_disconnect_cancel_watcher(disconnect_watcher) + if not completed and not cancel_event.is_set(): + cancel_event.set() + backend.reset_generation_state(cancel_event) + if gen is not None: + try: + await asyncio.to_thread(gen.close) + except (RuntimeError, ValueError): + pass + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(stream()) @@ -6675,6 +7075,10 @@ async def generate_audio( # the idle-stash restore runs here; switching TTS models is an explicit /load. await _maybe_auto_switch_model(_RELOAD_ONLY_MODEL, request, current_subject) + # Created before the backend pick so the GGUF lambda can close over it; the registration + # that arms it is below, once the model name is known. + _audio_cancel = threading.Event() + # Pick backend — both return (wav_bytes, sample_rate) llama_backend = get_llama_cpp_backend() if llama_backend.is_loaded and getattr(llama_backend, "_is_audio", False): @@ -6691,6 +7095,7 @@ async def generate_audio( min_p = payload.min_p, max_new_tokens = _effective_max_tokens(payload) or 2048, repetition_penalty = payload.repetition_penalty, + cancel_event = _audio_cancel, ) else: backend = get_inference_backend() @@ -6719,11 +7124,30 @@ async def generate_audio( # /audio/generate route and the chat-completions audio branches that delegate here. _fill_recommended_sampling_openai(payload, _audio_model_id) - try: - wav_bytes, sample_rate = await asyncio.to_thread(gen) - except Exception as e: - logger.error(f"Audio generation error: {e}", exc_info = True) - raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + # TTS holds the model for the whole request, so unregistered a non-forced swap counted zero + # generations and tore the model down mid-generation. The GGUF path observes the event; the + # subprocess backend blocks on its response queue with no cancel plumbing, so there it is + # only advisory -- which is why the swap drains are bounded. No cancel keys: /cancel + # addresses streams, and this route has none. + with _TrackedCancel( + _audio_cancel, + thread_id = getattr(payload, "thread_id", None), + model = model_name, + kind = "audio", + ): + # Stop in the UI aborts the fetch and nothing more, and this route has no cancel id to + # address, so without watching the disconnect llama-server kept generating for the rest + # of the request timeout after the chat had already reported it stopped. + _audio_watcher = asyncio.create_task(_await_disconnect_then_cancel(request, _audio_cancel)) + try: + wav_bytes, sample_rate = await asyncio.to_thread(gen) + except Exception as e: + if _audio_cancel.is_set(): + raise HTTPException(status_code = 499, detail = "Audio generation cancelled") + logger.error(f"Audio generation error: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + finally: + await _stop_local_disconnect_cancel_watcher(_audio_watcher) audio_b64 = base64.b64encode(wav_bytes).decode("ascii") return JSONResponse( @@ -8415,7 +8839,7 @@ async def openai_chat_completions( if payload.stream: _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() async def audio_input_stream(): @@ -8481,6 +8905,12 @@ async def openai_chat_completions( }, ) else: + # `stream` defaults to False, so this is the ordinary shape of an audio-input chat and it + # holds the worker for the whole request. Unregistered, a swap counted zero generations + # and cancelled it instead of 409ing (/unload runs no idle drain). + _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) + _tracker.__enter__() try: full_text = "" for chunk_text in audio_input_generate(): @@ -8494,6 +8924,9 @@ async def openai_chat_completions( except Exception as e: api_monitor.fail(monitor_id, _friendly_error(e)) raise + finally: + # Nested under the except arms too: api_monitor.fail() can throw, and a leaked entry 409s swaps. + _tracker.__exit__(None, None, None) api_monitor.set_reply(monitor_id, full_text) api_monitor.finish(monitor_id) response = ChatCompletion( @@ -8652,7 +9085,7 @@ async def openai_chat_completions( monitor_id = monitor_id, ) _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() try: return await _openai_passthrough_non_streaming( @@ -8889,13 +9322,37 @@ async def openai_chat_completions( _tool_sentinel = object() _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() async def gguf_tool_stream(): gen = None next_task = None stream_completed = False + # A call parked on the approval prompt is not decoding, so it gives its slot back; + # otherwise unanswered prompts hold every slot. + _parked = False + + async def _park_admission(on: bool, *, wait: bool = True): + nonlocal _parked + if on == _parked: + return + # This run's own lease, not a fresh lookup: queues are keyed by base_url and a + # reload mints a new port, so re-resolving could release someone else's slot. + lease = reservation.lease_nowait() + if lease is None: + return + if on: + lease.park() + elif wait: + # Resuming: park() may have handed our slot to a waiter, so wait for room instead + # of putting two holders on one slot. + await lease.unpark_async(cancel_event = cancel_event) + else: + # Tearing down; the lease is released separately. + lease.unpark() + _parked = on + disconnect_watcher = asyncio.create_task( _await_disconnect_then_cancel(request, cancel_event) ) @@ -8955,6 +9412,12 @@ async def openai_chat_completions( if event is _tool_sentinel: break + # Anything after the gated tool_start means the user answered. + if not ( + event["type"] == "tool_start" and event.get("awaiting_confirmation") + ): + await _park_admission(False) + if event["type"] == "heartbeat": # Tool-wrapper heartbeat while a server-side tool blocks; keeps SSE alive. yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE @@ -8993,6 +9456,8 @@ async def openai_chat_completions( yield chunk prev_text = "" reasoning_extractor = _new_chat_reasoning_extractor() + # Yielded just before the loop blocks on the user. + await _park_admission(bool(event.get("awaiting_confirmation"))) yield f"data: {json.dumps(event)}\n\n" continue @@ -9076,6 +9541,8 @@ async def openai_chat_completions( error_chunk = _openai_stream_error_chunk(e) yield _openai_stream_error_sse(error_chunk) finally: + # A disconnect mid-approval must not leave a slot parked. + await _park_admission(False, wait = False) try: if not stream_completed: cancel_event.set() @@ -9466,7 +9933,7 @@ async def openai_chat_completions( if _wants_multiple_choices(payload): raise _reject_unsupported_n("streaming GGUF chat completions") _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() try: reservation, admission_config = _openai_llama_admission_reserve( @@ -9785,7 +10252,7 @@ async def openai_chat_completions( raise _openai_admission_http_exception(exc, status_code = 429) _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() admission_lease = None admission_wait_started_at = None @@ -10227,7 +10694,7 @@ async def openai_chat_completions( _sf_tool_sentinel = object() _sf_cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _sf_tracker = _TrackedCancel(cancel_event, *_sf_cancel_keys) + _sf_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_sf_cancel_keys) _sf_tracker.__enter__() async def sf_tool_stream(): @@ -10256,11 +10723,11 @@ async def openai_chat_completions( while True: if cancel_event.is_set(): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) break if await request.is_disconnected(): cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") return @@ -10283,7 +10750,7 @@ async def openai_chat_completions( if event is _sf_tool_sentinel: break if isinstance(event, GenStreamError): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(event) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse( @@ -10378,16 +10845,16 @@ async def openai_chat_completions( except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") raise except GenStreamErrorRaised as exc: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}}) except Exception: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) # Generic wire message; full trace stays in the log (CWE-209: # transformers/torch errors may leak paths). logger.exception("safetensors tool stream error") @@ -10481,20 +10948,20 @@ async def openai_chat_completions( return _model_json_response(response) except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") raise except GenStreamErrorRaised as exc: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) raise HTTPException(status_code = 500, detail = _msg) except HTTPException as exc: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.fail(monitor_id, str(exc.detail)) raise except Exception: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) # CWE-209: generic detail; full trace in log. logger.exception("safetensors tool completion error") api_monitor.fail(monitor_id, "An internal error occurred.") @@ -10619,7 +11086,7 @@ async def openai_chat_completions( # ── Streaming response ──────────────────────────────────────── if payload.stream: _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() async def stream_chunks(): @@ -10646,7 +11113,7 @@ async def openai_chat_completions( gen = generate() while True: if cancel_event.is_set(): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) break # Stall keepalive (see safetensors tool stream) each window while # next(gen) runs in a worker. next(gen, _DONE) returns _DONE rather @@ -10666,7 +11133,7 @@ async def openai_chat_completions( if cumulative is _DONE: break if isinstance(cumulative, GenStreamError): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(cumulative) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse( @@ -10675,7 +11142,7 @@ async def openai_chat_completions( return if await request.is_disconnected(): cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") return new_text = cumulative[len(prev_text) :] @@ -10776,18 +11243,18 @@ async def openai_chat_completions( except asyncio.CancelledError: cancel_event.set() - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) api_monitor.finish(monitor_id, "cancelled") raise except GenStreamErrorRaised as exc: # Adapter-controlled (compare-mode) backend failure. Honor the # public flag so operational errors surface their real message. - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) yield _openai_stream_error_sse({"error": {"message": _msg, "type": "server_error"}}) except Exception as e: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) logger.error(f"Error during OpenAI streaming: {e}", exc_info = True) _msg = _friendly_error(e) api_monitor.fail(monitor_id, _msg) @@ -10826,11 +11293,17 @@ async def openai_chat_completions( # ── Non-streaming response ──────────────────────────────────── else: + # `stream` defaults to False, so this is the default shape of a standard (non-GGUF) chat and + # generate() holds the worker throughout. Unregistered, a swap cancelled this run rather + # than returning 409 (/unload runs no idle drain). + _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) + _tracker.__enter__() try: full_text = "" for token in generate(): if isinstance(token, GenStreamError): - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(token) api_monitor.fail(monitor_id, _msg) raise HTTPException(status_code = 500, detail = _msg) @@ -10937,15 +11410,18 @@ async def openai_chat_completions( except GenStreamErrorRaised as exc: # Adapter-controlled (compare-mode) backend failure. Honor the public # flag so operational errors surface their real message. - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) _msg = _friendly_gen_stream_error(exc) api_monitor.fail(monitor_id, _msg) raise HTTPException(status_code = 500, detail = _msg) except Exception as e: - backend.reset_generation_state() + backend.reset_generation_state(cancel_event) logger.error(f"Error during OpenAI completion: {e}", exc_info = True) api_monitor.fail(monitor_id, _friendly_error(e)) raise HTTPException(status_code = 500, detail = safe_error_detail(e)) + finally: + # Nested under the except arms too: reset_generation_state() can throw, and a leaked entry 409s swaps. + _tracker.__exit__(None, None, None) # ===================================================================== @@ -11399,10 +11875,11 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge target_url = f"{llama_backend.base_url}/v1/completions" is_stream = body.get("stream", False) prompt_text = _flatten_monitor_prompt(body.get("prompt", "")) + monitor_model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default") monitor_id = api_monitor.start( endpoint = request.url.path, method = request.method, - model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"), + model = monitor_model, prompt = prompt_text, context_length = llama_backend.context_length, subject = current_subject, @@ -11430,12 +11907,23 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge bytes_iter = None disconnect_event = threading.Event() disconnect_watcher = None + # This proxy relays straight from llama-server, so the swap gate has to see it: without an + # entry a non-forced /unload counts zero generations and tears the server down mid-response. + # Sharing disconnect_event lets a forced swap stop the relay through the check it already + # polls. Entered inside the body generator, so a response whose body never starts leaves + # nothing behind (see _responses_stream). No thread_id: public API surface, not a chat. + _tracker = _TrackedCancel(disconnect_event, model = monitor_model, kind = "completions") + _tracker.__enter__() try: req = client.build_request( "POST", target_url, json = body, headers = {"Connection": "close"} ) first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S - resp = await _send_stream_with_preheader_cancel(client, req, request = request) + # Same event the relay loop polls, so a forced swap ends the request during prefill + # instead of only once headers arrive. + resp = await _send_stream_with_preheader_cancel( + client, req, disconnect_event, request = request + ) if resp is None: api_monitor.finish(monitor_id, "cancelled") return @@ -11506,27 +11994,64 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge yield _openai_stream_error_sse_bytes(error_chunk) return finally: - await _aclose_stream_resources( - watchers = (disconnect_watcher,), - iterator = bytes_iter, - resp = resp, - client = client, - ) + # Nested so a close-time failure still unregisters; a phantom entry 409s swaps. + try: + await _aclose_stream_resources( + watchers = (disconnect_watcher,), + iterator = bytes_iter, + resp = resp, + client = client, + ) + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(_stream()) else: - try: - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _llama_non_streaming_generation_timeout(), + # ``stream`` defaults to false, so this common shape registers with the swap gate like the + # streaming branch: unregistered, a non-forced /unload counts zero generations and kills + # llama-server mid-request, and force_cancel_active has no event. Unpooled client so a + # cancel-close hits this call only. + _cancel_event = threading.Event() + _client = _cancelable_nonstreaming_client() + _tracker = _TrackedCancel(_cancel_event, model = monitor_model, kind = "completions") + _tracker.__enter__() + _cancel_watcher = asyncio.create_task( + _await_cancel_or_disconnect_then_close_client( + cancel_event = _cancel_event, + request = request, + client = _client, ) + ) + try: + try: + resp = await _client.post( + target_url, + json = body, + timeout = _llama_non_streaming_generation_timeout(), + ) + except httpx.RequestError: + # The watcher closed the client out from under the request: report the cancel, not a transport failure. + if _cancel_event.is_set(): + raise asyncio.CancelledError() + raise + if _cancel_event.is_set(): + raise asyncio.CancelledError() except asyncio.CancelledError: api_monitor.finish(monitor_id, "cancelled") raise except Exception as e: api_monitor.fail(monitor_id, _friendly_error(e)) raise + finally: + # Nested so a close-time failure still unregisters; a phantom entry 409s swaps. + try: + await _stop_local_disconnect_cancel_watcher(_cancel_watcher) + try: + await _client.aclose() + except Exception: + pass + finally: + _tracker.__exit__(None, None, None) if resp.status_code != 200: api_monitor.fail(monitor_id, resp.text[:500]) @@ -11622,18 +12147,54 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get subject = current_subject, ) - try: - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S, + # Same gate registration as the completions proxy: unregistered, a non-forced /unload counts + # zero generations and kills llama-server mid-embedding. Unpooled client so a cancel-close + # hits this call only. + _cancel_event = threading.Event() + _client = _cancelable_nonstreaming_client() + _tracker = _TrackedCancel( + _cancel_event, + model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"), + kind = "embeddings", + ) + _tracker.__enter__() + _cancel_watcher = asyncio.create_task( + _await_cancel_or_disconnect_then_close_client( + cancel_event = _cancel_event, + request = request, + client = _client, ) + ) + try: + try: + resp = await _client.post( + target_url, + json = body, + timeout = _DEFAULT_FIRST_TOKEN_TIMEOUT_S, + ) + except httpx.RequestError: + # The watcher closed the client out from under the request: report the cancel, not a transport failure. + if _cancel_event.is_set(): + raise asyncio.CancelledError() + raise + if _cancel_event.is_set(): + raise asyncio.CancelledError() except asyncio.CancelledError: api_monitor.finish(monitor_id, "cancelled") raise except Exception as exc: api_monitor.fail(monitor_id, _friendly_error(exc)) raise + finally: + # Nested so a close-time failure still unregisters; a phantom entry 409s swaps. + try: + await _stop_local_disconnect_cancel_watcher(_cancel_watcher) + try: + await _client.aclose() + except Exception: + pass + finally: + _tracker.__exit__(None, None, None) if resp.status_code != 200: api_monitor.fail(monitor_id, resp.text[:500]) else: @@ -12366,6 +12927,12 @@ async def _responses_stream( ) body["stream_options"] = {"include_usage": True} target_url = f"{llama_backend.base_url}/v1/chat/completions" + # The stream's own disconnect event, shared with the cancel/active-generation registries: + # this path decodes on llama-server, so a non-forced /unload must see it and refuse instead + # of tearing the server down mid-response. Entered inside the body generator below, so a + # response whose body never starts leaves nothing behind. + cancel_event = threading.Event() + _tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id) try: reservation, admission_config = _openai_llama_admission_reserve( request = request, @@ -12819,14 +13386,19 @@ async def _responses_stream( resp = None lines_iter = None disconnect_watcher = None - disconnect_event = threading.Event() + # Tracked per-run event: a client disconnect and a forced reload both land here. + disconnect_event = cancel_event try: req = client.build_request( "POST", target_url, json = body, headers = {"Connection": "close"} ) first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S try: - resp = await _send_stream_with_preheader_cancel(client, req, request = request) + # Same event the loop below polls: prefill can run for the whole first-token window, + # and only the send watcher can end it early. + resp = await _send_stream_with_preheader_cancel( + client, req, disconnect_event, request = request + ) if resp is None: api_monitor.finish(monitor_id, "cancelled") return @@ -13222,6 +13794,9 @@ async def _responses_stream( yield _sse("response.completed", completed_response) async def admitted_event_generator(): + # Register for the body's whole lifetime, admission wait included: the run holds a decode + # slot from here on, so /load and /unload must count it. __exit__ runs from the finally below. + _tracker.__enter__() lease = reservation.lease_nowait() admission_wait_started_at = None stream_started = False @@ -13238,11 +13813,14 @@ async def _responses_stream( completion_id = resp_id, level = "debug", ) + # The tracked event, not just the client socket: registered above, so a forced swap's + # cancel_all() reaches this run while it is still queued. Otherwise it takes a lease it was + # told to give up and the post-cancel drain waits out the round trip it just cancelled. async for wait_item in _openai_admission_wait_stream_chunks( reservation, admission_config, request = request, - cancel_event = None, + cancel_event = cancel_event, ): if isinstance(wait_item, str): yield wait_item @@ -13263,7 +13841,7 @@ async def _responses_stream( await _raise_if_openai_admission_cancelled( reservation, request = request, - cancel_event = None, + cancel_event = cancel_event, ) iterator = event_generator() stream_started = True @@ -13312,6 +13890,7 @@ async def _responses_stream( if not stream_started: api_monitor.finish(monitor_id, "cancelled") reservation.cancel() + _tracker.__exit__(None, None, None) async def _responses_admission_unstarted_cleanup() -> None: api_monitor.finish(monitor_id, "cancelled") @@ -13909,6 +14488,24 @@ async def anthropic_messages( cancel_event, ) + async def _tracked_anthropic_non_streaming(coro): + """Register a non-streaming /v1/messages run with the swap gate. + + `stream` defaults to false, so this is the route's common shape, and all + three helpers hold llama-server for the whole await. /unload runs no idle + drain, so unregistered a swap tore the server down mid-request; only the + streaming siblings registered. No cancel keys, unlike the streaming + tool/plain siblings: the gate reaches a run through the registry, and + keys would add a cancel surface to a public API. + """ + _tracker = _TrackedCancel(cancel_event, model = model_name, kind = "messages") + _tracker.__enter__() + try: + return await _monitored_anthropic(coro) + finally: + # _monitored_anthropic's bookkeeping can throw; a leaked entry 409s later swaps. + _tracker.__exit__(None, None, None) + # ── Admission control ───────────────────────────────────── # Bound concurrent llama-server generations to the backend's serving slots via a # FIFO queue keyed by base_url (shared with /v1/chat/completions, same slots). @@ -14080,7 +14677,9 @@ async def anthropic_messages( request = request, cancel_event = cancel_event, ) - monitored = await _monitored_anthropic(coro) + # Registered only once admitted: a queued request is not holding + # llama-server, so it has no business blocking a swap. + monitored = await _tracked_anthropic_non_streaming(coro) return monitored except LlamaAdmissionTimeout as exc: coro.close() @@ -14151,6 +14750,8 @@ async def anthropic_messages( disable_parallel_tool_use = _disable_parallel, auto_heal_tool_calls = payload.auto_heal_tool_calls, nudge_tool_calls = payload.nudge_tool_calls, + request = request, + cancel_event = cancel_event, ) ) @@ -14326,134 +14927,132 @@ async def _anthropic_tool_stream( ) async def _stream(): - emitter = AnthropicStreamEmitter() - for line in emitter.start(message_id, model_name, input_tokens = input_tokens): - yield line - - captured_finish_reason = None - # Whether the response currently ends on a pending tool_use block (the - # client must act → stop_reason "tool_use") as opposed to final text. - # The server may run a tool and then keep generating, which flips this - # back to False — that is an end_turn (or max_tokens) response. - ends_on_tool_use = False - tool_blocks_emitted = 0 - drop_until_tool_end = False - # Last drop-branch keepalive, seeded to stream start so a chatty tool busy - # past the stall window still gets a keepalive though its events are dropped. - _last_drop_keepalive = time.monotonic() - - gen = run_gen() - _next_task = None - # Watcher to cancel on disconnect: the in-loop poll fires only between - # events, so a mid-prefill disconnect would otherwise hold the decode slot. - disconnect_watcher = asyncio.create_task( - _await_disconnect_then_cancel(request, cancel_event) - ) + # The server-tool loop decodes on llama-server for its whole body, so without an entry a + # non-forced /unload saw zero generations and tore the server down mid-response. Entered + # inside the body generator so a response whose body never starts leaves nothing behind. + # No thread_id: public API surface. + _tracker = _TrackedCancel(cancel_event, model = model_name, kind = "messages") + _tracker.__enter__() try: - while True: - if cancel_event.is_set() or await request.is_disconnected(): - cancel_event.set() - return - # Stall keepalive (see GGUF tool stream): silent backend segments - # must not leave the SSE stream idle past proxy timeouts. - _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) - while True: - _done_tasks, _ = await asyncio.wait( - {_next_task}, - timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, - ) - if _done_tasks: - break - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - event = _next_task.result() - # Done; drop the reference so the finally-block drain no-ops. - _next_task = None - if event is _sentinel: - break - etype = event.get("type") - if etype == "heartbeat": - # Tool-wrapper heartbeat -> SSE keepalive, checked BEFORE the drop - # skip: a dropped tool still runs server-side and its events keep the - # stall keepalive from firing, so dropping heartbeats would go silent. - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - continue - if etype in ("tool_output", "tool_args"): - # Live stdout / arg streaming have no Anthropic Messages equivalent - # (the full call/result follow in tool_use / tool_result), so drop them. - # They keep the stall keepalive from firing, so a chatty tool would go - # silent past the ~100s proxy cap; emit a rate-limited keepalive instead. - _now = time.monotonic() - if _now - _last_drop_keepalive >= _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S: - _last_drop_keepalive = _now - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - continue - if drop_until_tool_end: - # disable_parallel_tool_use: skip every event until (and - # including) this dropped tool call's tool_end. - if etype == "tool_end": - drop_until_tool_end = False - continue - if etype == "metadata": - _fr = event.get("finish_reason") - if _fr is not None: - captured_finish_reason = _fr - # Strip leaked tool-call XML from content events first, so a - # content event that was purely tool XML doesn't count as text. - # Protected helper preserves rehearsal and balanced - # [TOOL_CALLS] trailing prose (raw _TOOL_XML_RE.sub corrupts both). - if etype == "content": - event = dict(event) - event["text"] = _strip_tool_xml_for_display( - event["text"], - auto_heal_tool_calls = True, - enabled_tool_names = _display_names, - ) - # disable_parallel_tool_use: keep only the first tool_use block, - # dropping every later tool_start and its paired tool_end (robust - # to empty tool-call ids — tracked by state, not id matching). - if etype == "tool_start": - if disable_parallel_tool_use and tool_blocks_emitted >= 1: - drop_until_tool_end = True - continue - ends_on_tool_use = True - elif etype == "tool_end": - tool_blocks_emitted += 1 - # A tool_end means Unsloth executed the tool server-side, so - # the response no longer ends on a pending client action. - # Without this, a server tool that produces no trailing text - # would be mislabeled stop_reason "tool_use", telling the - # client to run a tool Unsloth already ran. - ends_on_tool_use = False - elif etype == "content" and event.get("text"): - ends_on_tool_use = False - for line in emitter.feed(event): - yield line - except Exception as e: - logger.error("anthropic_messages stream error: %s", e) - # force = True so an unclassified mid-stream failure (llama-server crash, - # decode OOM, dropped socket) still emits an SSE error and returns, instead - # of a normal message_stop that masks a truncated turn as a clean finish. - _error_event = _anthropic_stream_error_event(e, force = True) - if _error_event is not None: - yield _error_event - return - finally: - await _stop_local_disconnect_cancel_watcher(disconnect_watcher) - # Drain a still-running next(gen) worker before closing, so a mid-prefill - # disconnect releases the thread/generator/tool resources. Closing first - # would race into ValueError('generator already executing'). - await _drain_pending_next_task(_next_task, cancel_event) - if gen is not None: - try: - await asyncio.to_thread(gen.close) - except (RuntimeError, ValueError): - pass + emitter = AnthropicStreamEmitter() + for line in emitter.start(message_id, model_name, input_tokens = input_tokens): + yield line - stop_reason = openai_finish_to_anthropic_stop( - captured_finish_reason, had_tool_calls = ends_on_tool_use - ) - for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): - yield line + captured_finish_reason = None + # Response ends on a pending tool_use block rather than final text; a server tool + # that keeps generating flips this back to False. + ends_on_tool_use = False + tool_blocks_emitted = 0 + drop_until_tool_end = False + # Last drop-branch keepalive, seeded to stream start so a chatty tool busy past the + # stall window still gets one though its events are dropped. + _last_drop_keepalive = time.monotonic() + + gen = run_gen() + _next_task = None + # Watcher to cancel on disconnect: the in-loop poll fires only between events, + # so a mid-prefill disconnect would hold the decode slot. + disconnect_watcher = asyncio.create_task( + _await_disconnect_then_cancel(request, cancel_event) + ) + try: + while True: + if cancel_event.is_set() or await request.is_disconnected(): + cancel_event.set() + return + # Stall keepalive (see GGUF tool stream): silent backend segments must not + # leave the SSE stream idle past proxy timeouts. + _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) + while True: + _done_tasks, _ = await asyncio.wait( + {_next_task}, + timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, + ) + if _done_tasks: + break + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + event = _next_task.result() + # Done; drop the reference so the finally-block drain no-ops. + _next_task = None + if event is _sentinel: + break + etype = event.get("type") + if etype == "heartbeat": + # Tool-wrapper heartbeat -> SSE keepalive, checked BEFORE the drop skip: + # a dropped tool still runs and suppresses the stall keepalive. + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + continue + if etype in ("tool_output", "tool_args"): + # No Anthropic Messages equivalent (the full call/result follow in tool_use / + # tool_result), so drop them. They suppress the stall keepalive, so emit a + # rate-limited one instead of going silent past the ~100s proxy cap. + _now = time.monotonic() + if _now - _last_drop_keepalive >= _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S: + _last_drop_keepalive = _now + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + continue + if drop_until_tool_end: + # disable_parallel_tool_use: skip every event until (and + # including) this dropped tool call's tool_end. + if etype == "tool_end": + drop_until_tool_end = False + continue + if etype == "metadata": + _fr = event.get("finish_reason") + if _fr is not None: + captured_finish_reason = _fr + # Strip leaked tool-call XML first, so a purely-tool-XML content event doesn't + # count as text. The protected helper keeps rehearsal and balanced + # [TOOL_CALLS] trailing prose, which a raw sub corrupts. + if etype == "content": + event = dict(event) + event["text"] = _strip_tool_xml_for_display( + event["text"], + auto_heal_tool_calls = True, + enabled_tool_names = _display_names, + ) + # disable_parallel_tool_use: keep only the first tool_use block, dropping + # later tool_start/tool_end pairs (by state, not id: ids may be empty). + if etype == "tool_start": + if disable_parallel_tool_use and tool_blocks_emitted >= 1: + drop_until_tool_end = True + continue + ends_on_tool_use = True + elif etype == "tool_end": + tool_blocks_emitted += 1 + # Unsloth ran the tool server-side, so the response no longer ends on a pending + # client action; otherwise stop_reason "tool_use" tells the client to run it again. + ends_on_tool_use = False + elif etype == "content" and event.get("text"): + ends_on_tool_use = False + for line in emitter.feed(event): + yield line + except Exception as e: + logger.error("anthropic_messages stream error: %s", e) + # force = True so an unclassified mid-stream failure emits an SSE error instead + # of a message_stop that masks a truncated turn as a clean finish. + _error_event = _anthropic_stream_error_event(e, force = True) + if _error_event is not None: + yield _error_event + return + finally: + await _stop_local_disconnect_cancel_watcher(disconnect_watcher) + # Drain a still-running next(gen) worker first, so a mid-prefill disconnect releases + # its resources; closing first races into 'already executing'. + await _drain_pending_next_task(_next_task, cancel_event) + if gen is not None: + try: + await asyncio.to_thread(gen.close) + except (RuntimeError, ValueError): + pass + + stop_reason = openai_finish_to_anthropic_stop( + captured_finish_reason, had_tool_calls = ends_on_tool_use + ) + for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): + yield line + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(_stream()) @@ -14477,75 +15076,81 @@ async def _anthropic_plain_stream( input_tokens = await asyncio.to_thread(llama_backend.count_chat_tokens, openai_messages) async def _stream(): - emitter = AnthropicStreamEmitter() - for line in emitter.start(message_id, model_name, input_tokens = input_tokens): - yield line - - captured_finish_reason = None - - gen = run_gen() - _next_task = None - # Watcher to cancel on disconnect: the in-loop poll fires only between - # chunks, so a mid-prefill disconnect would otherwise hold the decode slot. - disconnect_watcher = asyncio.create_task( - _await_disconnect_then_cancel(request, cancel_event) - ) + # Registered like the tool stream above: this default /v1/messages path decodes on + # llama-server, so without an entry a non-forced /unload tore it down mid-response. + _tracker = _TrackedCancel(cancel_event, model = model_name, kind = "messages") + _tracker.__enter__() try: - while True: - if cancel_event.is_set() or await request.is_disconnected(): - cancel_event.set() - return - # Stall keepalive (see Anthropic tool stream) each window while - # next(gen) runs in a worker. - _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) - while True: - _done_tasks, _ = await asyncio.wait( - {_next_task}, - timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, - ) - if _done_tasks: - break - yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE - cumulative = _next_task.result() - # Done; drop the reference so the finally-block drain no-ops. - _next_task = None - if cumulative is _sentinel: - break - if isinstance(cumulative, dict): - if cumulative.get("type") == "metadata": - _fr = cumulative.get("finish_reason") - if _fr is not None: - captured_finish_reason = _fr - for line in emitter.feed(cumulative): - yield line - continue - # Plain generator yields cumulative text strings - for line in emitter.feed({"type": "content", "text": cumulative}): - yield line - except Exception as e: - logger.error("anthropic_messages stream error: %s", e) - # force = True so an unclassified mid-stream failure (llama-server crash, - # decode OOM, dropped socket) still emits an SSE error and returns, instead - # of a normal message_stop that masks a truncated turn as a clean finish. - _error_event = _anthropic_stream_error_event(e, force = True) - if _error_event is not None: - yield _error_event - return - finally: - await _stop_local_disconnect_cancel_watcher(disconnect_watcher) - # Drain a still-running next(gen) worker before closing, so a mid-prefill - # disconnect releases the thread/generator/model resources. Closing first - # would race into ValueError('generator already executing'). - await _drain_pending_next_task(_next_task, cancel_event) - if gen is not None: - try: - await asyncio.to_thread(gen.close) - except (RuntimeError, ValueError): - pass + emitter = AnthropicStreamEmitter() + for line in emitter.start(message_id, model_name, input_tokens = input_tokens): + yield line - stop_reason = openai_finish_to_anthropic_stop(captured_finish_reason, had_tool_calls = False) - for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): - yield line + captured_finish_reason = None + + gen = run_gen() + _next_task = None + # Watcher to cancel on disconnect: the in-loop poll fires only between chunks, + # so a mid-prefill disconnect would hold the decode slot. + disconnect_watcher = asyncio.create_task( + _await_disconnect_then_cancel(request, cancel_event) + ) + try: + while True: + if cancel_event.is_set() or await request.is_disconnected(): + cancel_event.set() + return + # Stall keepalive each window while next(gen) runs in a worker. + _next_task = asyncio.create_task(asyncio.to_thread(next, gen, _sentinel)) + while True: + _done_tasks, _ = await asyncio.wait( + {_next_task}, + timeout = _LOCAL_TOOL_STREAM_STALL_KEEPALIVE_S, + ) + if _done_tasks: + break + yield _OPENAI_PASSTHROUGH_SSE_KEEPALIVE + cumulative = _next_task.result() + # Done; drop the reference so the finally-block drain no-ops. + _next_task = None + if cumulative is _sentinel: + break + if isinstance(cumulative, dict): + if cumulative.get("type") == "metadata": + _fr = cumulative.get("finish_reason") + if _fr is not None: + captured_finish_reason = _fr + for line in emitter.feed(cumulative): + yield line + continue + # Plain generator yields cumulative text strings + for line in emitter.feed({"type": "content", "text": cumulative}): + yield line + except Exception as e: + logger.error("anthropic_messages stream error: %s", e) + # force = True so an unclassified mid-stream failure emits an SSE error instead + # of a message_stop that masks a truncated turn as a clean finish. + _error_event = _anthropic_stream_error_event(e, force = True) + if _error_event is not None: + yield _error_event + return + finally: + await _stop_local_disconnect_cancel_watcher(disconnect_watcher) + # Drain a still-running next(gen) worker first, so a mid-prefill disconnect releases + # its resources; closing first races into 'already executing'. + await _drain_pending_next_task(_next_task, cancel_event) + if gen is not None: + try: + await asyncio.to_thread(gen.close) + except (RuntimeError, ValueError): + pass + + stop_reason = openai_finish_to_anthropic_stop( + captured_finish_reason, had_tool_calls = False + ) + for line in emitter.finish(stop_reason = stop_reason, stop_sequence = None): + yield line + finally: + _tracker.__exit__(None, None, None) return _sse_streaming_response(_stream()) @@ -14981,10 +15586,23 @@ async def _anthropic_passthrough_stream( # cancel_id mirrors the OpenAI passthrough so a per-run cancel POST # works without the caller having to know the local message_id. - _tracker = _TrackedCancel(cancel_event, cancel_id, session_id, message_id) - _tracker.__enter__() + # No thread_id: public API surface, but still registered so a reload cannot yank + # llama-server out from under it. Built here, entered below inside _stream(). + _tracker = _TrackedCancel( + cancel_event, + cancel_id, + session_id, + message_id, + model = model_name, + kind = "messages", + ) async def _stream(): + # Entered inside the body, not eagerly: aclose() runs no body on a generator + # that never started, so a client that drops first would leave the run + # registered until restart, 409-ing every swap. Ahead of the first yield, so + # the opening lines are covered as well. + _tracker.__enter__() emitter = AnthropicPassthroughEmitter() # Promote text-form tool calls (declared client tools only) into # tool_use blocks; verbatim behavior when healing is off or no tools. @@ -15162,8 +15780,16 @@ async def _anthropic_passthrough_non_streaming( disable_parallel_tool_use = False, auto_heal_tool_calls = None, nudge_tool_calls = None, + request: Optional[Request] = None, + cancel_event = None, ): - """Non-streaming client-side pass-through.""" + """Non-streaming client-side pass-through. + + Both POSTs run on a per-request client so a Stop or a forced swap can close + it and interrupt them. The pooled ``nonstreaming_client()`` cannot be closed + without disturbing unrelated calls, which left this path registered with the + swap gate but deaf to the event it registered. + """ target_url = f"{llama_backend.base_url}/v1/chat/completions" body = _build_passthrough_payload( openai_messages, @@ -15181,138 +15807,162 @@ async def _anthropic_passthrough_non_streaming( backend_ctx = llama_backend.context_length, ) - try: - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _llama_non_streaming_generation_timeout(), - ) - except httpx.ConnectError as exc: - # Nothing was returned yet, so retry once against the respawned server's - # new port; the nudge retry below then reuses the same fresh URL. - retry_url = await _anthropic_passthrough_retry_url(llama_backend, exc) - if retry_url is None: - raise - target_url = retry_url - resp = await nonstreaming_client().post( - target_url, - json = body, - timeout = _llama_non_streaming_generation_timeout(), + _client = _cancelable_nonstreaming_client() + _cancel_watcher = asyncio.create_task( + _await_cancel_or_disconnect_then_close_client( + cancel_event = cancel_event, + request = request, + client = _client, ) + ) - if resp.status_code != 200: - raise HTTPException( - status_code = resp.status_code, - detail = _friendly_upstream_error(resp.text[:500]), - ) - - data = resp.json() - # tool_choice arrives here already converted to the OpenAI shape. - _allowed_tools = heal_gate(auto_heal_tool_calls, openai_tools, tool_choice) - - # Opt-in single-retry nudge (mirrors the OpenAI passthrough): the model - # tried to call a tool but nothing usable came out; re-ask once with the - # prompt prefix intact so llama-server's KV cache is reused. - if ( - _allowed_tools - and nudge_enabled(nudge_tool_calls) - and nudge_should_retry(data, _allowed_tools, openai_tools) - ): - retry_body = { - **body, - "messages": [*body.get("messages", []), *nudge_messages(data, _allowed_tools)], - } + async def _post(payload_body): + nonlocal target_url try: - retry_resp = await nonstreaming_client().post( + return await _client.post( target_url, - json = retry_body, + json = payload_body, + timeout = _llama_non_streaming_generation_timeout(), + ) + except httpx.RequestError as exc: + # The watcher closes the client to break a blocked POST, so a transport error + # with the event set is the cancel, not a failure. + if cancel_event is not None and cancel_event.is_set(): + raise asyncio.CancelledError() + # Nothing was returned yet, so retry once against the respawned server's + # new port; the nudge retry below then reuses the same fresh URL. + retry_url = ( + await _anthropic_passthrough_retry_url(llama_backend, exc) + if isinstance(exc, httpx.ConnectError) + else None + ) + if retry_url is None: + raise + target_url = retry_url + return await _client.post( + target_url, + json = payload_body, timeout = _llama_non_streaming_generation_timeout(), ) - if retry_resp.status_code == 200: - retry_data = retry_resp.json() - if response_has_promotable_calls(retry_data, _allowed_tools, openai_tools): - data = retry_data - except (httpx.RequestError, ValueError) as exc: - logger.warning("tool-call nudge retry failed; keeping original: %s", exc) - choice = (data.get("choices") or [{}])[0] - message = choice.get("message") or {} - finish_reason = choice.get("finish_reason") + try: + resp = await _post(body) - healing_active = bool(_allowed_tools) - healed_events = ( - heal_openai_message_events(message, _allowed_tools, openai_tools) - if healing_active - else None - ) + if resp.status_code != 200: + raise HTTPException( + status_code = resp.status_code, + detail = _friendly_upstream_error(resp.text[:500]), + ) - content_blocks = [] - tool_calls = [] - if healed_events: - emitted_tool_uses = 0 - for kind, value in healed_events: - if kind == "text": - text = str(value).strip() + data = resp.json() + # tool_choice arrives here already converted to the OpenAI shape. + _allowed_tools = heal_gate(auto_heal_tool_calls, openai_tools, tool_choice) + + # Opt-in single-retry nudge (mirrors the OpenAI passthrough): the tool call came out + # unusable; re-ask with the prompt prefix intact so the KV cache is reused. + if ( + _allowed_tools + and nudge_enabled(nudge_tool_calls) + and nudge_should_retry(data, _allowed_tools, openai_tools) + ): + retry_body = { + **body, + "messages": [*body.get("messages", []), *nudge_messages(data, _allowed_tools)], + } + try: + retry_resp = await _post(retry_body) + if retry_resp.status_code == 200: + retry_data = retry_resp.json() + if response_has_promotable_calls(retry_data, _allowed_tools, openai_tools): + data = retry_data + except (httpx.RequestError, ValueError) as exc: + logger.warning("tool-call nudge retry failed; keeping original: %s", exc) + + choice = (data.get("choices") or [{}])[0] + message = choice.get("message") or {} + finish_reason = choice.get("finish_reason") + + healing_active = bool(_allowed_tools) + healed_events = ( + heal_openai_message_events(message, _allowed_tools, openai_tools) + if healing_active + else None + ) + + content_blocks = [] + tool_calls = [] + if healed_events: + emitted_tool_uses = 0 + for kind, value in healed_events: + if kind == "text": + text = str(value).strip() + if text: + content_blocks.append(AnthropicResponseTextBlock(text = text)) + continue + if disable_parallel_tool_use and emitted_tool_uses >= 1: + continue + fn = value.get("function") or {} + try: + args = json.loads(fn.get("arguments", "{}")) + except json.JSONDecodeError: + args = {} + tool_calls.append(value) + emitted_tool_uses += 1 + content_blocks.append( + AnthropicResponseToolUseBlock( + id = anthropic_tool_use_id(value.get("id")), + name = fn.get("name", ""), + input = args, + ) + ) + else: + text = message.get("content") or "" + if text: + # Keep unpromoted bytes when healing is active; legacy stripping is only for opted-out + # or no-client-tool requests. The protected helper preserves rehearsal and + # balanced [TOOL_CALLS] prose, gated on the declared tools so an inactive + # NAME[ARGS]{...} example is kept. + if not healing_active: + text = _strip_tool_xml_for_display( + text, + auto_heal_tool_calls = True, + enabled_tool_names = _display_tool_name_gate(openai_tools), + ) + text = text.strip() if text: content_blocks.append(AnthropicResponseTextBlock(text = text)) - continue - if disable_parallel_tool_use and emitted_tool_uses >= 1: - continue - fn = value.get("function") or {} - try: - args = json.loads(fn.get("arguments", "{}")) - except json.JSONDecodeError: - args = {} - tool_calls.append(value) - emitted_tool_uses += 1 - content_blocks.append( - AnthropicResponseToolUseBlock( - id = anthropic_tool_use_id(value.get("id")), - name = fn.get("name", ""), - input = args, - ) - ) - else: - text = message.get("content") or "" - if text: - # Keep unpromoted bytes when healing is active; legacy stripping is - # only for opted-out or no-client-tool requests. Protected helper (not - # raw _TOOL_XML_RE.sub): preserves rehearsal and balanced - # [TOOL_CALLS] trailing prose, gated on the declared tools so an - # inactive NAME[ARGS]{...} example in the final text is kept. - if not healing_active: - text = _strip_tool_xml_for_display( - text, - auto_heal_tool_calls = True, - enabled_tool_names = _display_tool_name_gate(openai_tools), - ) - text = text.strip() - if text: - content_blocks.append(AnthropicResponseTextBlock(text = text)) - tool_calls = message.get("tool_calls") or [] - if disable_parallel_tool_use and len(tool_calls) > 1: - tool_calls = tool_calls[:1] - for tc in tool_calls: - fn = tc.get("function") or {} - try: - args = json.loads(fn.get("arguments", "{}")) - except json.JSONDecodeError: - args = {} - content_blocks.append( - AnthropicResponseToolUseBlock( - id = anthropic_tool_use_id(tc.get("id")), - name = fn.get("name", ""), - input = args, + tool_calls = message.get("tool_calls") or [] + if disable_parallel_tool_use and len(tool_calls) > 1: + tool_calls = tool_calls[:1] + for tc in tool_calls: + fn = tc.get("function") or {} + try: + args = json.loads(fn.get("arguments", "{}")) + except json.JSONDecodeError: + args = {} + content_blocks.append( + AnthropicResponseToolUseBlock( + id = anthropic_tool_use_id(tc.get("id")), + name = fn.get("name", ""), + input = args, + ) ) - ) - stop_reason = openai_finish_to_anthropic_stop(finish_reason, had_tool_calls = bool(tool_calls)) + stop_reason = openai_finish_to_anthropic_stop( + finish_reason, had_tool_calls = bool(tool_calls) + ) - usage = data.get("usage") or {} - return _anthropic_message_json_response( - message_id, model_name, content_blocks, stop_reason, usage - ) + usage = data.get("usage") or {} + return _anthropic_message_json_response( + message_id, model_name, content_blocks, stop_reason, usage + ) + finally: + await _stop_local_disconnect_cancel_watcher(_cancel_watcher) + try: + await _client.aclose() + except Exception: + pass # ===================================================================== @@ -15694,7 +16344,7 @@ async def _openai_passthrough_stream( monitor_id: Optional[str] = None, ): _cancel_keys = (payload.cancel_id, payload.session_id, completion_id) - _tracker = _TrackedCancel(cancel_event, *_cancel_keys) + _tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys) _tracker.__enter__() try: reservation, admission_config = _openai_llama_admission_reserve( diff --git a/studio/backend/run.py b/studio/backend/run.py index 5dfab9346a..8ef1ac06b8 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -1377,13 +1377,21 @@ def _apply_cli_tool_policy(enable_tools: "Optional[bool]") -> None: set_tool_policy(enable_tools) +# Mirror unsloth_cli/commands/studio.py's _PARALLEL_*: the admission queue caps concurrent +# chats at the slot count, so a direct launch matches the CLI (VRAM fit may still cut it +# back). Defined above run_server() so embedders that omit it do not serialise every chat. +_PARALLEL_MIN = 1 +_PARALLEL_MAX = 64 +_PARALLEL_DEFAULT_PLAIN = 4 + + def run_server( host: str = "127.0.0.1", port: int = 8888, frontend_path: Path = _DEFAULT_FRONTEND_PATH, silent: bool = False, api_only: bool = False, - llama_parallel_slots: int = 1, + llama_parallel_slots: int = _PARALLEL_DEFAULT_PLAIN, cloudflare: "Optional[bool]" = None, secure: bool = False, enable_tools: "Optional[bool]" = None, @@ -1399,7 +1407,8 @@ def run_server( frontend_path: Path to frontend build directory (optional) silent: Suppress startup messages api_only: API server only, no frontend (for Tauri desktop app) - llama_parallel_slots: parallel slots for llama-server + llama_parallel_slots: parallel slots for llama-server (default + _PARALLEL_DEFAULT_PLAIN, matching the CLI entry points) cloudflare: opt in to the public Cloudflare HTTPS tunnel for a wildcard bind. Tri-state: None (unset) and False both mean off; True enables it. --secure implies it (True) and rejects an explicit False. @@ -1817,13 +1826,6 @@ def run_server( return app -# Mirror unsloth_cli/commands/studio.py's _PARALLEL_*. Default 1 is for direct -# backend launches; `unsloth studio run` always passes its own value (4). -_PARALLEL_MIN = 1 -_PARALLEL_MAX = 64 -_PARALLEL_DEFAULT_PLAIN = 1 - - def _build_arg_parser(): """Build the backend CLI argument parser. @@ -1918,7 +1920,7 @@ def _build_arg_parser(): default = _PARALLEL_DEFAULT_PLAIN, help = ( f"llama-server parallel decode slots ({_PARALLEL_MIN}..{_PARALLEL_MAX}). " - f"Default {_PARALLEL_DEFAULT_PLAIN}; `unsloth studio run` uses 4." + f"Default {_PARALLEL_DEFAULT_PLAIN}." ), ) return parser diff --git a/studio/backend/state/active_generations.py b/studio/backend/state/active_generations.py new file mode 100644 index 0000000000..d1f2812c59 --- /dev/null +++ b/studio/backend/state/active_generations.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Registry of in-flight chat generations, keyed by conversation. + +New Chat leaves the previous conversation streaming, so /load and /unload need +to know which chats a reload would interrupt: they refuse with 409 unless the +caller opts in to cancelling them, and GET /inference/active-generations lets +the UI name them. A frontend guard alone would miss a second tab or a REST call. + +Entries hold the same threading.Event as the per-run cancel registry in +routes/inference.py, so cancel_all() closes each generation's own upstream +stream and never signals llama-server itself. + +A plain dict plus a threading.Lock: no signals, no process groups, no event loop +affinity, so it behaves identically on Linux, macOS, Windows and WSL. +""" + +from __future__ import annotations + +import threading +import time +import uuid +from typing import Any, Optional + +# handle id -> entry. Keyed by handle, not thread_id: a tool continuation can register +# before the previous leg unregisters, and one key would drop the other. +_ACTIVE: dict[str, dict[str, Any]] = {} +_LOCK = threading.Lock() + + +class ActiveGeneration: + """Registers one in-flight generation for the duration of the block. + + Each __enter__ mints its own handle, so overlapping uses never clobber. + """ + + __slots__ = ("thread_id", "cancel_event", "model", "kind", "_handle") + + def __init__( + self, + cancel_event: threading.Event, + *, + thread_id: Optional[str] = None, + model: Optional[str] = None, + kind: str = "chat", + ): + self.thread_id = thread_id or None + self.cancel_event = cancel_event + self.model = model or None + self.kind = kind + self._handle: Optional[str] = None + + def __enter__(self) -> "ActiveGeneration": + self._handle = uuid.uuid4().hex + with _LOCK: + _ACTIVE[self._handle] = { + "handle": self._handle, + "thread_id": self.thread_id, + "model": self.model, + "kind": self.kind, + "started_at": time.time(), + "event": self.cancel_event, + } + return self + + def __exit__(self, *exc) -> bool: + handle, self._handle = self._handle, None + if handle is not None: + with _LOCK: + _ACTIVE.pop(handle, None) + return False + + +def snapshot() -> list[dict[str, Any]]: + """In-flight generations, newest last. Drops the Event: this is a response.""" + with _LOCK: + entries = list(_ACTIVE.values()) + entries.sort(key = lambda e: e["started_at"]) + return [ + { + "handle": e["handle"], + "thread_id": e["thread_id"], + "model": e["model"], + "kind": e["kind"], + "started_at": e["started_at"], + } + for e in entries + ] + + +def active_thread_ids() -> list[str]: + """Distinct conversation ids with a generation in flight, in start order. + + A first turn that races persistence has no thread id yet: count() sees it, + this cannot name it. + """ + seen: list[str] = [] + for e in snapshot(): + tid = e["thread_id"] + if tid and tid not in seen: + seen.append(tid) + return seen + + +def count() -> int: + """Number of generations currently in flight.""" + with _LOCK: + return len(_ACTIVE) + + +def cancel_all() -> int: + """Signal every in-flight generation to stop. Returns how many were signalled. + + Only sets the cancel events; each stream tears itself down. Entries are + removed by their own __exit__, so one mid-cleanup is neither lost nor double + counted. + """ + with _LOCK: + events = [e["event"] for e in _ACTIVE.values()] + for ev in events: + try: + ev.set() + except Exception: + pass + return len(events) + + +def cancel_thread(thread_id: str) -> int: + """Signal only the generations belonging to ``thread_id``.""" + if not thread_id: + return 0 + with _LOCK: + events = [e["event"] for e in _ACTIVE.values() if e["thread_id"] == thread_id] + for ev in events: + try: + ev.set() + except Exception: + pass + return len(events) + + +def reset_for_tests() -> None: + """Drop every entry. Test-only; never called from request paths.""" + with _LOCK: + _ACTIVE.clear() diff --git a/studio/backend/tests/test_active_generations.py b/studio/backend/tests/test_active_generations.py new file mode 100644 index 0000000000..aa087fe4ea --- /dev/null +++ b/studio/backend/tests/test_active_generations.py @@ -0,0 +1,2635 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Parallel chats: the active-generation registry and the model-swap gate. + +A load/unload has to know which streaming chats it would interrupt. Everything +under test is a dict + threading.Lock, so this passes on every platform. +""" + +import os +import sys +import threading + +import pytest + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + +from state import active_generations + + +@pytest.fixture(autouse = True) +def _clean_registry(): + active_generations.reset_for_tests() + yield + active_generations.reset_for_tests() + + +# ── registry ────────────────────────────────────────────────────────── + + +def test_registry_starts_empty(): + assert active_generations.count() == 0 + assert active_generations.snapshot() == [] + assert active_generations.active_thread_ids() == [] + + +def test_entry_lives_only_for_the_block(): + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1", model = "m"): + assert active_generations.count() == 1 + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.count() == 0 + assert active_generations.active_thread_ids() == [] + + +def test_entry_is_removed_even_when_the_block_raises(): + ev = threading.Event() + with pytest.raises(RuntimeError): + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + raise RuntimeError("stream blew up") + assert active_generations.count() == 0 + + +def test_overlapping_runs_on_one_thread_both_register(): + # A tool continuation registers its next leg before the previous unwinds. + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t1"): + assert active_generations.count() == 2 + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.count() == 1 + assert active_generations.count() == 0 + + +def test_snapshot_is_json_safe_and_ordered_by_start(): + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "first", model = "m1"): + with active_generations.ActiveGeneration(b, thread_id = "second", model = "m2"): + snap = active_generations.snapshot() + assert [e["thread_id"] for e in snap] == ["first", "second"] + # The threading.Event must not leak into an HTTP response body. + assert all("event" not in e for e in snap) + assert {"handle", "thread_id", "model", "kind", "started_at"} == set(snap[0]) + + +def test_thread_ids_are_deduped_and_skip_unnamed_runs(): + a, b, c = threading.Event(), threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t1"): + # A brand-new chat whose first turn races persistence has no id yet. + with active_generations.ActiveGeneration(c, thread_id = None): + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.count() == 3 + + +# ── cancellation ────────────────────────────────────────────────────── + + +def test_cancel_all_sets_every_event(): + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + assert active_generations.cancel_all() == 2 + assert a.is_set() and b.is_set() + + +def test_cancel_all_on_an_empty_registry_is_a_no_op(): + assert active_generations.cancel_all() == 0 + + +def test_cancel_thread_leaves_siblings_alone(): + # Per-thread Stop: the rest keep generating, llama-server is untouched. + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + assert active_generations.cancel_thread("t1") == 1 + assert a.is_set() + assert not b.is_set() + + +def test_cancel_thread_with_no_match_is_a_no_op(): + a = threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + assert active_generations.cancel_thread("nope") == 0 + assert active_generations.cancel_thread("") == 0 + assert not a.is_set() + + +def test_cancel_does_not_unregister_entries(): + # __exit__ owns removal, so a generation mid-cleanup is not lost. + a = threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + active_generations.cancel_all() + assert active_generations.count() == 1 + + +# ── concurrency ─────────────────────────────────────────────────────── + + +def test_registry_survives_concurrent_register_unregister(): + errors: list[BaseException] = [] + barrier = threading.Barrier(8) + + def worker(i: int) -> None: + try: + barrier.wait(timeout = 10) + for _ in range(50): + with active_generations.ActiveGeneration(threading.Event(), thread_id = f"t{i}"): + active_generations.snapshot() + except BaseException as exc: # noqa: BLE001 - surfaced via assert below + errors.append(exc) + + threads = [threading.Thread(target = worker, args = (i,)) for i in range(8)] + for t in threads: + t.start() + for t in threads: + t.join(timeout = 30) + + assert errors == [] + assert active_generations.count() == 0 + + +# ── the model-swap gate ─────────────────────────────────────────────── + + +# The gate lives in routes.inference, which pulls the whole inference stack. +def _route_gate(): + pytest.importorskip("fastapi", reason = "inference stack not installed") + routes_inference = pytest.importorskip( + "routes.inference", reason = "inference stack not installed" + ) + return routes_inference._raise_or_cancel_active_generations + + +@pytest.fixture +def gate(): + return _route_gate() + + +def test_gate_allows_a_swap_when_nothing_is_generating(gate): + assert gate(force = False, action = "Loading a model") == 0 + + +def test_gate_refuses_with_409_and_names_the_chats(gate): + from fastapi import HTTPException + + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + with pytest.raises(HTTPException) as exc: + gate(force = False, action = "Loading a model") + assert exc.value.status_code == 409 + detail = exc.value.detail + assert detail["error"] == "active_generations" + assert detail["running"] == 2 + assert detail["thread_ids"] == ["t1", "t2"] + # Refusing must not cancel anything. + assert not a.is_set() and not b.is_set() + + +def test_gate_message_is_singular_for_one_chat(gate): + from fastapi import HTTPException + + with active_generations.ActiveGeneration(threading.Event(), thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + gate(force = False, action = "Unloading the model") + message = exc.value.detail["message"] + assert "1 chat that is still generating" in message + assert "Unloading the model" in message + + +def test_gate_force_cancels_and_returns_the_count(gate): + a, b = threading.Event(), threading.Event() + with active_generations.ActiveGeneration(a, thread_id = "t1"): + with active_generations.ActiveGeneration(b, thread_id = "t2"): + assert gate(force = True, action = "Loading a model") == 2 + assert a.is_set() and b.is_set() + + +def test_gate_force_with_nothing_running_is_a_no_op(gate): + assert gate(force = True, action = "Loading a model") == 0 + + +# ── the route wiring ────────────────────────────────────────────────── + + +def test_tracked_cancel_registers_the_thread_for_its_block(): + # The single place a generation is recorded, so every streaming path gets it. + _route_gate() + from routes.inference import _TrackedCancel + + ev = threading.Event() + tracker = _TrackedCancel(ev, "cancel-1", thread_id = "t1", model = "m") + tracker.__enter__() + try: + assert active_generations.active_thread_ids() == ["t1"] + assert active_generations.snapshot()[0]["model"] == "m" + finally: + tracker.__exit__(None, None, None) + assert active_generations.count() == 0 + + +def test_tracked_cancel_shares_its_event_with_the_registry(): + # Reusing the per-run event is what keeps a forced reload off llama-server. + _route_gate() + from routes.inference import _TrackedCancel + + ev = threading.Event() + tracker = _TrackedCancel(ev, "cancel-1", thread_id = "t1") + tracker.__enter__() + try: + active_generations.cancel_all() + assert ev.is_set() + finally: + tracker.__exit__(None, None, None) + + +def _stub_load_route(monkeypatch, *, active_model_name): + """Point POST /load at an in-memory safetensors backend. + + active_model_name == the requested path makes the request idempotent, so + _load_model_impl takes its already_loaded fast return. + """ + from types import SimpleNamespace + + import routes.inference as inf_mod + + monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", lambda: None) + monkeypatch.setattr(inf_mod, "validate_extra_args", lambda args: []) + monkeypatch.setattr( + inf_mod, + "resolve_effective_chat_template_override", + lambda model_identifier = None, user_override = None: None, + ) + monkeypatch.setattr(inf_mod, "load_inference_config", lambda name: {}) + monkeypatch.setattr( + inf_mod, + "_detect_safetensors_features", + lambda backend, template, tools = None: { + "supports_reasoning": False, + "reasoning_style": "enable_thinking", + "reasoning_effort_levels": [], + "reasoning_always_on": False, + "supports_preserve_thinking": False, + "supports_tools": False, + }, + ) + monkeypatch.setattr(inf_mod, "_resolve_loaded_trust_remote_code", lambda *a, **k: False) + monkeypatch.setattr( + inf_mod, + "get_inference_backend", + lambda: SimpleNamespace(active_model_name = active_model_name, models = {}), + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace(is_loaded = False, hf_variant = None, model_identifier = None), + ) + return inf_mod + + +def test_idempotent_load_neither_refuses_nor_cancels_running_chats(monkeypatch): + # Re-applying the resident model hits already_loaded: no llama-server touch, no 409, no stopped chats. + _route_gate() + import asyncio + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/A") + + for force in (False, True): + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = asyncio.run( + inf_mod.load_model( + LoadRequest(model_path = "org/A", force_cancel_active = force), + object(), + "tester", + ) + ) + assert response.status == "already_loaded" + assert not ev.is_set() + + +def test_a_real_reload_still_refuses_while_chats_stream(monkeypatch): + # A load that would really replace the model still 409s and names the chats. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run(inf_mod.load_model(LoadRequest(model_path = "org/A"), object(), "tester")) + assert exc.value.status_code == 409 + assert exc.value.detail["thread_ids"] == ["t1"] + assert not ev.is_set() + + +def test_a_forced_load_that_fails_preflight_leaves_the_chats_alone(monkeypatch): + # Preflight can still reject after the user confirms, so cancelling first ends chats for nothing. + _route_gate() + import asyncio + import contextlib + + from fastapi import HTTPException + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + # Stands in for any preflight refusal; a None here is the route's own 400. + monkeypatch.setattr(inf_mod.ModelConfig, "from_identifier", staticmethod(lambda **kwargs: None)) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.load_model( + LoadRequest(model_path = "org/A", force_cancel_active = True), + object(), + "tester", + ) + ) + # The load was rejected, so the chat must still be streaming. + assert not ev.is_set() + assert active_generations.count() == 1 + assert exc.value.status_code == 400 + + +def _stub_standard_load_route(monkeypatch): + """Drive _load_model_impl down the Unsloth path as far as the pre-teardown drain.""" + import contextlib + from types import SimpleNamespace + + import routes.inference as inf_mod + + real_sidecar_check = inf_mod._raise_if_sidecar_swap_in_progress + _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + # _stub_load_route neutralises the sidecar guard; this test is about it. + monkeypatch.setattr(inf_mod, "_raise_if_sidecar_swap_in_progress", real_sidecar_check) + monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + monkeypatch.setattr(inf_mod, "_mlx_distributed_launch_detected", lambda: False) + monkeypatch.setattr( + inf_mod.ModelConfig, + "from_identifier", + staticmethod( + lambda **kwargs: SimpleNamespace( + is_gguf = False, + identifier = "org/A", + display_name = "A", + is_vision = False, + gguf_hf_repo = None, + gguf_variant = None, + ) + ), + ) + monkeypatch.setattr(inf_mod, "_effective_load_in_4bit", lambda config, requested: False) + monkeypatch.setattr(inf_mod, "_resolve_inherited_extra_args", lambda *a, **k: None) + monkeypatch.setattr(inf_mod, "_guard_chat_load_against_training", lambda *a, **k: None) + return inf_mod + + +def test_a_sidecar_swap_reserved_during_the_drain_never_strands_cancelled_chats(monkeypatch): + # A sidecar install can reserve the swap window during the pre-teardown drain, so the recheck + # after it is the last rejection point and must precede the cancel, else chats die for nothing. + _route_gate() + import asyncio + import time + from types import SimpleNamespace + + from fastapi import HTTPException + + from core.inference import llama_keepwarm as kw + from models.inference import LoadRequest + + import utils.transformers_version as tv + + inf_mod = _stub_standard_load_route(monkeypatch) + reserved = {"v": False} + monkeypatch.setattr(tv, "sidecar_swap_in_progress", lambda: reserved["v"]) + + # Two tracked requests; the install reserves the window mid-drain when the uncancellable one ends. + monkeypatch.setattr(kw, "_inflight", 2) + + def _installer(): + time.sleep(0.10) + kw._inflight = 1 # the non-cancellable request finished ... + reserved["v"] = True # ... and an install reserved the swap window + time.sleep(0.35) + kw._inflight = 0 # the chat's own request drains last + + thread = threading.Thread(target = _installer, daemon = True) + ev = threading.Event() + try: + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + thread.start() + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.load_model( + LoadRequest(model_path = "org/A", force_cancel_active = True), + SimpleNamespace( + app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1)) + ), + "tester", + ) + ) + # Rejected, so the chat traded for a model it never got must still stream. + assert not ev.is_set() + assert active_generations.count() == 1 + assert exc.value.status_code == 409 + assert "transformers installation" in str(exc.value.detail) + finally: + thread.join(timeout = 5) + kw._inflight = 0 + + +def _stub_unload_backends(monkeypatch, *, llama, backend): + """Point the /unload route at in-memory backends.""" + import routes.inference as inf_mod + from core.inference import llama_keepwarm as kw + + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: llama) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "is_registered_native_path_label", lambda *a: False) + monkeypatch.setattr(kw, "note_model_unloaded", lambda: None) + return inf_mod, kw + + +def test_unload_rechecks_active_generations_under_the_lifecycle_gate(monkeypatch): + # Without the recheck, a chat that starts while this queues on the gate is torn down mid-stream. + _route_gate() + import asyncio + from types import SimpleNamespace + + from fastapi import HTTPException + + from models.inference import UnloadRequest + + torn_down: list[str] = [] + inf_mod, kw = _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = True, + model_identifier = "org/A-GGUF", + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: None, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + started = active_generations.ActiveGeneration(ev, thread_id = "t1") + + async def drive(): + # A load holds the lifecycle gate, so the unload queues behind it. + kw._lifecycle_lock.acquire() + task = asyncio.create_task( + inf_mod.unload_model(UnloadRequest(model_path = "org/A-GGUF"), "tester") + ) + entered = False + try: + await asyncio.sleep(0.1) # the route is polling the gate + started.__enter__() # a chat starts in the meantime + entered = True + finally: + kw._lifecycle_lock.release() + try: + return await asyncio.wait_for(task, timeout = 5) + finally: + if entered: + started.__exit__(None, None, None) + + with pytest.raises(HTTPException) as exc: + asyncio.run(drive()) + + # 409, not the catch-all 500 the route wraps unexpected failures in. + assert exc.value.status_code == 409 + assert exc.value.detail["error"] == "active_generations" + assert torn_down == [] + assert not ev.is_set() + + +def _run_unload( + inf_mod, + monkeypatch, + *, + loaded_gguf, + requested, + force, + torn_down, + unload_model = None, +): + """Drive POST /unload against a backend pair with ``loaded_gguf`` resident. + + ``unload_model`` overrides the GGUF teardown so a caller can observe what the + world looked like at the moment of teardown, not just afterwards. + """ + import asyncio + from types import SimpleNamespace + + from models.inference import UnloadRequest + + _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = True, + model_identifier = loaded_gguf, + unload_model = unload_model or (lambda: torn_down.append("gguf")), + ), + # Nothing on the standard backend: the GGUF above is what is resident. + backend = SimpleNamespace( + get_loading_model = lambda: None, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + return asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = requested, force_cancel_active = force), "tester" + ) + ) + + +def test_forced_unload_of_a_stale_model_path_leaves_the_chats_alone(monkeypatch): + # Eject naming a model another tab swapped out: a no-op success; cancelling first loses runs. + _route_gate() + import routes.inference as inf_mod + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/B-GGUF", # what the other tab actually loaded + requested = "org/A-GGUF", # this tab's stale idea of it + force = True, + torn_down = torn_down, + ) + assert not ev.is_set() + assert active_generations.count() == 1 + # The resident GGUF was never touched, so nothing was worth cancelling. + assert "gguf" not in torn_down + assert response.status == "unloaded" + + +def test_forced_unload_of_the_loaded_model_still_stops_its_chats(monkeypatch): + # A real unload must still cancel, or llama-server goes down mid-stream. + _route_gate() + import routes.inference as inf_mod + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + ) + assert ev.is_set() + assert torn_down == ["gguf"] + assert response.status == "unloaded" + + +def test_forced_unload_lets_the_cancelled_chats_unwind_before_teardown(monkeypatch): + # /unload used to tear down right after the cancel, so a stream told to stop but not yet + # finished lost its server. Assert the count hits zero BEFORE unload_model runs. + _route_gate() + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + inflight = {"n": 1} + seen = {} + + def _count(current_request_counted = True, *, include_pending = True): + # Unwinds one poll after the cancel, like a stream noticing its event. + if inflight["n"] > 0: + inflight["n"] -= 1 + return inflight["n"] + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0) + + torn_down: list[str] = [] + ev = threading.Event() + + def _record_teardown(): + seen["inflight_at_teardown"] = inflight["n"] + torn_down.append("gguf") + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + unload_model = _record_teardown, + ) + assert ev.is_set() + + assert torn_down == ["gguf"] + assert seen["inflight_at_teardown"] == 0 + assert response.status == "unloaded" + + +def test_unload_drains_on_the_middleware_count_not_just_the_registry(monkeypatch): + # A request past the middleware but not yet at its _TrackedCancel is counted but unregistered, so + # the drain reads the middleware count, not "did we cancel anything": one poll on a quiet server. + _route_gate() + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + polls = {"n": 0} + + def _count(current_request_counted = True, *, include_pending = True): + polls["n"] += 1 + return 0 + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + + torn_down: list[str] = [] + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + ) + assert torn_down == ["gguf"] + # Polled, but returned on the first read rather than waiting anything out. + assert polls["n"] == 1 + assert response.status == "unloaded" + + +def test_unforced_unload_of_a_stale_model_path_is_still_a_no_op(monkeypatch): + # Same stale Eject unforced: it reaches no teardown, so refusing strands the stale tab's selection. + _route_gate() + import routes.inference as inf_mod + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/B-GGUF", # what the other tab actually loaded + requested = "org/A-GGUF", # this tab's stale idea of it + force = False, + torn_down = torn_down, + ) + assert not ev.is_set() + assert active_generations.count() == 1 + # The resident GGUF was untouched; only the standard backend's stale-path no-op ran. + assert torn_down == ["unsloth"] + assert response.status == "unloaded" + + +def test_unforced_unload_of_the_loaded_model_still_refuses_while_chats_stream(monkeypatch): + # The stale skip above must not disarm the gate for a real replacement. + _route_gate() + import routes.inference as inf_mod + + from fastapi import HTTPException + + torn_down: list[str] = [] + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = False, + torn_down = torn_down, + ) + assert exc.value.status_code == 409 + assert exc.value.detail["thread_ids"] == ["t1"] + assert torn_down == [] + assert not ev.is_set() + + +def test_unforced_unload_still_refuses_while_a_gguf_load_is_in_flight(monkeypatch): + # A stale tab's Eject naming the PREVIOUS model while a different one loads. The GGUF branch + # evicts a live llama-server, so a chat on the previous model must get the 409, not be killed. + _route_gate() + import asyncio + from types import SimpleNamespace + + from fastapi import HTTPException + + from models.inference import UnloadRequest + + torn_down: list[str] = [] + inf_mod, _kw = _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = False, # spawned, health check not passed: mid-load + model_identifier = "org/B-GGUF", + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: None, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = "org/A-GGUF", force_cancel_active = False), + "tester", + ) + ) + assert exc.value.status_code == 409 + assert torn_down == [] + assert not ev.is_set() + + +def test_cancelling_an_in_flight_standard_load_is_not_refused_by_the_chat_gate(monkeypatch): + # The real cancelLoading shape: unforced /unload naming the still-LOADING model. It replaces + # nothing, so it cannot interrupt a chat and must not 409 (the frontend would drop the error). + _route_gate() + import asyncio + from types import SimpleNamespace + + from models.inference import UnloadRequest + + cancelled: list[str] = [] + torn_down: list[str] = [] + inf_mod, _kw = _stub_unload_backends( + monkeypatch, + # Nothing on llama-server: the load in flight is a safetensors one. + llama = SimpleNamespace( + is_active = False, + is_loaded = False, + model_identifier = None, + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: "org/B", + cancel_load = lambda path: bool(cancelled.append(path)) or True, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = "org/B", force_cancel_active = False), "tester" + ) + ) + # The chat on the previous model is untouched: the load never reached it. + assert not ev.is_set() + assert active_generations.count() == 1 + assert response.status == "unloaded" + assert cancelled == ["org/B"] + assert torn_down == [] + + +def test_cancelling_an_in_flight_gguf_load_is_not_refused_by_the_chat_gate(monkeypatch): + # Same cancelLoading shape on the GGUF fast path: killing that child ends a load, not a chat. + _route_gate() + import asyncio + from types import SimpleNamespace + + from models.inference import UnloadRequest + + torn_down: list[str] = [] + inf_mod, _kw = _stub_unload_backends( + monkeypatch, + llama = SimpleNamespace( + is_active = True, + is_loaded = False, # spawned, health check not passed: mid-load + model_identifier = "org/B-GGUF", + unload_model = lambda: torn_down.append("gguf"), + ), + backend = SimpleNamespace( + get_loading_model = lambda: None, + active_model_name = None, + models = {}, + unload_model = lambda path: torn_down.append("unsloth"), + ), + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + response = asyncio.run( + inf_mod.unload_model( + UnloadRequest(model_path = "org/B-GGUF", force_cancel_active = False), "tester" + ) + ) + assert not ev.is_set() + assert active_generations.count() == 1 + assert response.status == "unloaded" + assert torn_down == ["gguf"] + + +def _install_responses_stream_mock(monkeypatch, chunks): + """Point the direct /v1/responses GGUF pass-through at an in-process + llama-server. Mirrors the harness in test_responses_tool_passthrough.py.""" + import json + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + def handler(request): + content = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + content += "data: [DONE]\n\n" + return httpx.Response( + 200, + content = content.encode(), + headers = {"content-type": "text/event-stream"}, + ) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + is_vision = False, + context_length = 4096, + base_url = "http://llama.test", + supports_reasoning = True, + reasoning_always_on = False, + _request_reasoning_kwargs = ( + lambda enable_thinking = None, reasoning_effort = None, preserve_thinking = None: None + ), + ), + ) + return inf_mod + + +class _NeverDisconnectedRequest: + async def is_disconnected(self): + return False + + +def test_direct_responses_stream_is_visible_to_the_swap_gate(monkeypatch): + # /v1/responses streams straight to llama-server; unregistered, a non-forced /unload tore it down. + _route_gate() + import asyncio + + from models.inference import ChatMessage, ResponsesRequest + + inf_mod = _install_responses_stream_mock( + monkeypatch, [{"choices": [{"delta": {"content": "33"}}]}] + ) + payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF") + messages = [ChatMessage(role = "user", content = "hi")] + seen = {} + + async def run(): + response = await inf_mod._responses_stream(payload, messages, _NeverDisconnectedRequest()) + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + # And it unregisters, or one Codex call would 409 every later reload. + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_direct_responses_stream(monkeypatch): + # The registered event must be the one the stream watches, or a forced reload kills a live decode. + _route_gate() + import asyncio + + from models.inference import ChatMessage, ResponsesRequest + + inf_mod = _install_responses_stream_mock( + monkeypatch, + [ + {"choices": [{"delta": {"content": "3"}}]}, + {"choices": [{"delta": {"content": "3"}}]}, + ], + ) + payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF") + messages = [ChatMessage(role = "user", content = "hi")] + + async def run(): + response = await inf_mod._responses_stream(payload, messages, _NeverDisconnectedRequest()) + iterator = response.body_iterator + chunks = [await iterator.__anext__()] + assert active_generations.cancel_all() == 1 + async for chunk in iterator: + chunks.append(chunk) + return "".join(c.decode() if isinstance(c, bytes) else c for c in chunks) + + body = asyncio.run(run()) + + # Cancelled mid-stream: the run ends without a completed envelope. + assert "response.completed" not in body + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_responses_stream_still_queued_for_a_slot(monkeypatch): + # The run registers before it holds a decode slot, so cancel_all() must reach it while queued in + # admission; watching only the client socket lets it open a generation the swap already revoked. + _route_gate() + import asyncio + + from core.inference import llama_admission + from models.inference import ChatMessage, ResponsesRequest + + for name in ( + llama_admission.ADMISSION_CONTROL_ENV, + llama_admission.ADMISSION_QUEUE_TIMEOUT_ENV, + llama_admission.ADMISSION_KEEPALIVE_INTERVAL_ENV, + llama_admission.ADMISSION_MAX_QUEUE_ENV, + ): + monkeypatch.delenv(name, raising = False) + + inf_mod = _install_responses_stream_mock( + monkeypatch, [{"choices": [{"delta": {"content": "33"}}]}] + ) + payload = ResponsesRequest(input = "hi", stream = True, model = "org/M-GGUF") + messages = [ChatMessage(role = "user", content = "hi")] + + llama_admission.reset_llama_admission_queues() + try: + + async def run(): + # Hold the backend's only decode slot so the run below has to queue. + queue = llama_admission.get_llama_admission_queue("http://llama.test") + holder = queue.reserve(capacity = 1, config = llama_admission.LlamaAdmissionConfig()) + assert holder.lease_nowait() is not None + response = await inf_mod._responses_stream( + payload, messages, _NeverDisconnectedRequest() + ) + chunks = [] + + async def drain(): + async for chunk in response.body_iterator: + chunks.append(chunk) + + task = asyncio.create_task(drain()) + for _ in range(500): + if active_generations.count() == 1: + break + await asyncio.sleep(0.01) + assert active_generations.count() == 1, "the queued run never registered" + assert active_generations.cancel_all() == 1 + # Unbounded queue by default: without the tracked event this never returns while the slot is held. + await asyncio.wait_for(task, timeout = 5) + return chunks + + chunks = asyncio.run(run()) + finally: + llama_admission.reset_llama_admission_queues() + + body = "".join(c.decode() if isinstance(c, bytes) else c for c in chunks) + # It gave up its place instead of taking the slot: no upstream call, no envelope. + assert "response.created" not in body + assert active_generations.count() == 0 + + +def _install_completions_stream_mock(monkeypatch, events): + """Point the /v1/completions proxy at an in-process llama-server.""" + import json + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + def handler(request): + # One network chunk per SSE event: the relay polls its cancel flag between upstream chunks. + async def _chunks(): + for event in events: + yield f"data: {json.dumps(event)}\n\n".encode() + yield b"data: [DONE]\n\n" + + return httpx.Response( + 200, + content = _chunks(), + headers = {"content-type": "text/event-stream"}, + ) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + context_length = 4096, + base_url = "http://llama.test", + model_identifier = "org/M-GGUF", + ), + ) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + + async def _no_auto_switch(request, current_subject): + return await request.json() + + monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch) + return inf_mod + + +class _CompletionsRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/completions reads.""" + + def __init__(self, body): + from types import SimpleNamespace + + self._body = body + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/completions") + + async def json(self): + return self._body + + +def test_completions_proxy_stream_is_visible_to_the_swap_gate(monkeypatch): + # /v1/completions relays from llama-server with no idle drain; unregistered, /unload tore it down. + _route_gate() + import asyncio + + inf_mod = _install_completions_stream_mock(monkeypatch, [{"choices": [{"text": "33"}]}]) + request = _CompletionsRequest( + {"prompt": "hi", "stream": True, "model": "org/M-GGUF", "max_tokens": 8} + ) + seen = {} + + async def run(): + response = await inf_mod.openai_completions(request, "tester") + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + # And it unregisters, or one completion would 409 every later reload. + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_completions_proxy_stream(monkeypatch): + # The registered event must be the one the relay watches, or a forced reload kills a live decode. + _route_gate() + import asyncio + + inf_mod = _install_completions_stream_mock( + monkeypatch, + [{"choices": [{"text": "3"}]}, {"choices": [{"text": "3"}]}], + ) + request = _CompletionsRequest( + {"prompt": "hi", "stream": True, "model": "org/M-GGUF", "max_tokens": 8} + ) + + async def run(): + response = await inf_mod.openai_completions(request, "tester") + iterator = response.body_iterator + chunks = [await iterator.__anext__()] + assert active_generations.cancel_all() == 1 + async for chunk in iterator: + chunks.append(chunk) + return b"".join(c if isinstance(c, bytes) else c.encode() for c in chunks) + + body = asyncio.run(run()) + + # Stopped after the first event instead of relaying the rest. + assert body.count(b'"text"') == 1 + assert active_generations.count() == 0 + + +def test_completions_proxy_non_stream_is_visible_to_the_swap_gate(monkeypatch): + # ``stream`` defaults to false, so the non-streaming branch is the common shape and holds + # llama-server throughout: unregistered, /unload counts zero and force_cancel_active has no event. + _route_gate() + import asyncio + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + seen = {} + + def handler(request): + # Sampled mid-flight: exactly the window a concurrent /unload would tear down in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + # And the gate must reach this run, not just see it. + seen["cancelled"] = active_generations.cancel_all() + return httpx.Response(200, json = {"id": "cmpl-x", "choices": [{"text": "33"}]}) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + # The pooled client too, so a route that took no per-request one still reaches this transport. + monkeypatch.setattr( + inf_mod, "nonstreaming_client", lambda: real_async_client(transport = transport) + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + context_length = 4096, + base_url = "http://llama.test", + model_identifier = "org/M-GGUF", + ), + ) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + + async def _no_auto_switch(request, current_subject): + return await request.json() + + monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch) + + request = _CompletionsRequest({"prompt": "hi", "model": "org/M-GGUF", "max_tokens": 8}) + + with pytest.raises(asyncio.CancelledError): + asyncio.run(inf_mod.openai_completions(request, "tester")) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # And it unregisters, or one completion would 409 every later reload. + assert active_generations.count() == 0 + + +class _EmbeddingsRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/embeddings reads.""" + + def __init__(self, body): + from types import SimpleNamespace + + self._body = body + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/embeddings") + self.state = SimpleNamespace(skip_api_monitor = True) + + async def json(self): + return self._body + + +def test_embeddings_proxy_is_visible_to_the_swap_gate(monkeypatch): + # /v1/embeddings holds llama-server for its whole HTTP call: unregistered, a non-forced /unload + # counts zero and kills the server mid-request (only /load waits on the middleware count). + _route_gate() + import asyncio + from types import SimpleNamespace + + import httpx + + import routes.inference as inf_mod + + seen = {} + + def handler(request): + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + return httpx.Response(200, json = {"data": [{"embedding": [0.1, 0.2]}]}) + + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod.httpx, + "AsyncClient", + lambda *a, **kw: real_async_client(transport = transport, timeout = kw.get("timeout", 600)), + ) + monkeypatch.setattr( + inf_mod, "nonstreaming_client", lambda: real_async_client(transport = transport) + ) + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = True, + context_length = 4096, + base_url = "http://llama.test", + model_identifier = "org/M-GGUF", + ), + ) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + + async def _no_auto_switch(request, current_subject): + return await request.json() + + monkeypatch.setattr(inf_mod, "_auto_switch_from_request_body", _no_auto_switch) + + request = _EmbeddingsRequest({"input": "hi", "model": "org/M-GGUF"}) + + with pytest.raises(asyncio.CancelledError): + asyncio.run(inf_mod.openai_embeddings(request, "tester")) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # And it unregisters, or one embedding would 409 every later reload. + assert active_generations.count() == 0 + + +def test_active_generations_redacts_native_model_paths(monkeypatch): + # The legacy stream records active_model_name verbatim (an absolute path locally) and is the only + # place that serialises it: redact like the error paths so a remote client cannot learn host paths. + _route_gate() + import asyncio + import threading + from types import SimpleNamespace + + import routes.inference as inf_mod + from utils.native_path_leases import _remember_native_path_for_redaction + + secret_path = "/home/somebody/models/private-model.gguf" + _remember_native_path_for_redaction(secret_path, "private-model.gguf") + + request = SimpleNamespace(app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 4))) + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: SimpleNamespace()) + + with active_generations.ActiveGeneration(threading.Event(), thread_id = "t1", model = secret_path): + body = asyncio.run(inf_mod.get_active_generations(request, "tester")) + + assert body["count"] == 1 + assert secret_path not in str(body) + assert body["active"][0]["model"] == "" + + +def test_legacy_generate_stream_is_visible_to_the_swap_gate(monkeypatch): + # The legacy /generate/stream decodes on the standard backend throughout: unregistered it passed + # the advertised 409 gate then blocked on the generation lock, and a forced swap had no event. + _route_gate() + import asyncio + from types import SimpleNamespace + + import routes.inference as inf_mod + from models.inference import GenerateRequest + + seen = {} + + def _fake_generate_chat_response(**kwargs): + # Sampled mid-generation: exactly the window an /unload would land in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + yield "hello" + yield "world" + + backend = SimpleNamespace( + active_model_name = "org/M", + models = {"org/M": {}}, + generate_chat_response = lambda **kw: _fake_generate_chat_response(**kw), + reset_generation_state = lambda *a: None, + resize_image = lambda img: img, + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend) + + async def _drain(): + response = await inf_mod.generate_stream( + GenerateRequest(messages = [{"role": "user", "content": "hi"}]), + _NeverDisconnectedRequest(), + current_subject = "tester", + ) + async for _ in response.body_iterator: + pass + + asyncio.run(_drain()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M" + assert seen["cancelled"] == 1 + # And it unregisters, or one legacy stream would 409 every later reload. + assert active_generations.count() == 0 + + +def _anthropic_stream_args(chunks): + """(request, cancel_event, run_gen) for the local Anthropic stream helpers.""" + cancel_event = threading.Event() + + def run_gen(): + def _gen(): + for chunk in chunks: + if cancel_event.is_set(): + return + yield chunk + + return _gen() + + return _NeverDisconnectedRequest(), cancel_event, run_gen + + +def test_local_anthropic_plain_stream_is_visible_to_the_swap_gate(monkeypatch): + # Only the client-tool pass-through registered, so the no-tool /v1/messages path died mid-response. + _route_gate() + import asyncio + + import routes.inference as inf_mod + + request, cancel_event, run_gen = _anthropic_stream_args(["3", "33"]) + seen = {} + + async def run(): + response = await inf_mod._anthropic_plain_stream( + request, cancel_event, run_gen, "msg_1", "org/M-GGUF" + ) + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert active_generations.count() == 0 + + +def test_forced_reload_stops_a_local_anthropic_plain_stream(monkeypatch): + # The event registered has to be the one the decode loop watches. + _route_gate() + import asyncio + + import routes.inference as inf_mod + + request, cancel_event, run_gen = _anthropic_stream_args(["3", "33", "333"]) + + async def run(): + response = await inf_mod._anthropic_plain_stream( + request, cancel_event, run_gen, "msg_1", "org/M-GGUF" + ) + iterator = response.body_iterator + chunks = [await iterator.__anext__()] + assert active_generations.cancel_all() == 1 + async for chunk in iterator: + chunks.append(chunk) + return "".join(c.decode() if isinstance(c, bytes) else c for c in chunks) + + body = asyncio.run(run()) + + assert cancel_event.is_set() + # Cancelled mid-stream: no clean message_stop envelope. + assert "message_stop" not in body + assert active_generations.count() == 0 + + +def test_local_anthropic_tool_stream_is_visible_to_the_swap_gate(monkeypatch): + # Same gap on the server-tool path (enable_tools / Anthropic server tools). + _route_gate() + import asyncio + + import routes.inference as inf_mod + + request, cancel_event, run_gen = _anthropic_stream_args( + [{"type": "content", "text": "3"}, {"type": "content", "text": "33"}] + ) + seen = {} + + async def run(): + response = await inf_mod._anthropic_tool_stream( + request, cancel_event, run_gen, "msg_1", "org/M-GGUF" + ) + iterator = response.body_iterator + await iterator.__anext__() + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + async for _ in iterator: + pass + + asyncio.run(run()) + + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert active_generations.count() == 0 + + +def test_load_and_unload_requests_default_to_not_cancelling(): + pytest.importorskip("pydantic", reason = "pydantic not installed") + from models.inference import LoadRequest, UnloadRequest + + assert LoadRequest(model_path = "m").force_cancel_active is False + assert UnloadRequest(model_path = "m").force_cancel_active is False + assert LoadRequest(model_path = "m", force_cancel_active = True).force_cancel_active is True + + +def _parallel_constants(path: str) -> dict: + """Read the _PARALLEL_* constants from a file's source. + + Importing run.py would drag in the whole server to read three integers. + """ + import ast + + with open(path, encoding = "utf-8") as f: + tree = ast.parse(f.read()) + found = {} + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + for target in node.targets: + name = getattr(target, "id", "") + if name.startswith("_PARALLEL_") and isinstance(node.value, ast.Constant): + found[name] = node.value.value + return found + + +def test_studio_defaults_to_more_than_one_decode_slot(): + # With one slot the admission queue serialises every chat. + consts = _parallel_constants(os.path.join(_backend, "run.py")) + + assert consts["_PARALLEL_DEFAULT_PLAIN"] > 1 + assert consts["_PARALLEL_MIN"] <= consts["_PARALLEL_DEFAULT_PLAIN"] <= consts["_PARALLEL_MAX"] + + +def test_cli_and_backend_parallel_defaults_agree(): + # argparse and the typer CLI are separate entry points into the same server. + backend = _parallel_constants(os.path.join(_backend, "run.py")) + cli_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(_backend))), + "unsloth_cli", + "commands", + "studio.py", + ) + cli = _parallel_constants(cli_path) + + assert cli["_PARALLEL_DEFAULT_PLAIN"] == backend["_PARALLEL_DEFAULT_PLAIN"] + + +def _run_server_parallel_default(path: str, consts: dict): + """Resolve run_server()'s llama_parallel_slots default from run.py's source.""" + import ast + + with open(path, encoding = "utf-8") as f: + tree = ast.parse(f.read()) + for node in tree.body: + if not isinstance(node, ast.FunctionDef) or node.name != "run_server": + continue + args = node.args.args + defaults = node.args.defaults + # defaults align with the tail of the positional arg list. + for arg, default in zip(args[len(args) - len(defaults) :], defaults): + if arg.arg != "llama_parallel_slots": + continue + if isinstance(default, ast.Constant): + return default.value + if isinstance(default, ast.Name): + return consts.get(default.id) + return None + return None + + +def test_run_server_default_matches_the_cli_parallel_default(): + # colab.py omits llama_parallel_slots, so the signature default is what Colab runs with. + run_path = os.path.join(_backend, "run.py") + consts = _parallel_constants(run_path) + + default = _run_server_parallel_default(run_path, consts) + + assert default is not None, "run_server() must keep a llama_parallel_slots default" + assert default == consts["_PARALLEL_DEFAULT_PLAIN"] + assert default > 1 + + +def test_colab_launcher_inherits_the_parallel_default(): + # Guard the inheritance itself: an explicit 1 here would resurrect the bug. + import ast + + colab_path = os.path.join(_backend, "colab.py") + with open(colab_path, encoding = "utf-8") as f: + tree = ast.parse(f.read()) + consts = _parallel_constants(os.path.join(_backend, "run.py")) + + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "run_server" + ] + assert calls, "colab.py must still launch the backend through run_server()" + for call in calls: + for kw in call.keywords: + if kw.arg != "llama_parallel_slots": + continue + value = kw.value.value if isinstance(kw.value, ast.Constant) else None + assert ( + value is None or value > 1 + ), "colab.py pins llama_parallel_slots to 1; Colab chats would serialise" + # Whether pinned or inherited, Colab must end up with more than one slot. + assert consts["_PARALLEL_DEFAULT_PLAIN"] > 1 + + +# ── the point of no return ──────────────────────────────────────────── + + +def test_a_forced_load_that_loses_to_a_sidecar_install_leaves_the_chats_alone(monkeypatch): + # The destructive cancel is the point of no return: nothing after it may reject the load. A sidecar + # install can reserve the window during preflight, so its recheck must run before, not after. + _route_gate() + import asyncio + import contextlib + from types import SimpleNamespace + + from fastapi import HTTPException + + from models.inference import LoadRequest + + inf_mod = _stub_load_route(monkeypatch, active_model_name = "org/OTHER") + monkeypatch.setattr(inf_mod, "_hf_offline_if_dns_dead", contextlib.nullcontext) + monkeypatch.setattr( + inf_mod.ModelConfig, + "from_identifier", + staticmethod( + lambda **kwargs: SimpleNamespace( + is_gguf = False, + identifier = "org/A", + display_name = "A", + is_vision = False, + is_lora = False, + path = None, + ) + ), + ) + monkeypatch.setattr(inf_mod, "_mlx_distributed_launch_detected", lambda: False) + monkeypatch.setattr(inf_mod, "_guard_chat_load_against_training", lambda *a, **k: None) + monkeypatch.setattr(inf_mod, "_resolve_inherited_extra_args", lambda *a, **k: None) + + # The two route-level checks pass, every check after them 409s. + seen = {"calls": 0} + + def _sidecar_reserved_during_preflight(): + seen["calls"] += 1 + if seen["calls"] > 2: + raise HTTPException( + status_code = 409, + detail = "A transformers installation is in progress. Retry when it completes.", + ) + + monkeypatch.setattr( + inf_mod, "_raise_if_sidecar_swap_in_progress", _sidecar_reserved_during_preflight + ) + + fastapi_request = SimpleNamespace( + app = SimpleNamespace(state = SimpleNamespace(llama_parallel_slots = 1)) + ) + + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.load_model( + LoadRequest( + model_path = "org/A", + load_in_4bit = False, + force_cancel_active = True, + ), + fastapi_request, + "tester", + ) + ) + # The load was rejected, so the chat must still be streaming. + assert not ev.is_set() + assert active_generations.count() == 1 + assert exc.value.status_code == 409 + + +def test_anthropic_passthrough_registers_nothing_until_its_body_starts(): + # A pass-through response whose body never starts must leave both registries clean: a never-started + # async generator runs no body code (PEP 342), so an eagerly entered tracker never unregisters. + _route_gate() + import asyncio + import inspect + from types import SimpleNamespace + + from starlette.requests import ClientDisconnect + + import routes.inference as inf_mod + + llama_backend = SimpleNamespace( + base_url = "http://127.0.0.1:8080", + context_length = 4096, + count_chat_tokens = lambda messages, _unused, tools: 7, + ) + + async def _build(): + return await inf_mod._anthropic_passthrough_stream( + SimpleNamespace(), + threading.Event(), + llama_backend, + [{"role": "user", "content": "hi"}], + [], + 0.7, + 0.9, + 40, + 128, + "msg_1", + "org/A", + session_id = "s1", + cancel_id = "c1", + ) + + # Built and abandoned, as when the request task is cancelled before Starlette calls the response. + asyncio.run(_build()) + assert active_generations.count() == 0 + assert not inf_mod._CANCEL_REGISTRY + + # The client is gone at header time, so the first send fails and the body generator never runs. + async def _drive(): + response = await _build() + + async def _receive(): + return {"type": "http.disconnect"} + + async def _send(message): + raise OSError("client disconnected") + + with pytest.raises(ClientDisconnect): + await response({"type": "http"}, _receive, _send) + + asyncio.run(_drive()) + assert active_generations.count() == 0 + assert not inf_mod._CANCEL_REGISTRY + + # Still tracked once the body runs: the enter stays inside the generator, under the finally. + src = inspect.getsource(inf_mod._anthropic_passthrough_stream) + assert src.index("async def _stream()") < src.index("_tracker.__enter__()") + assert src.index("_tracker.__enter__()") < src.index("_tracker.__exit__(None, None, None)") + + +def test_audio_generation_is_visible_to_the_swap_gate(monkeypatch): + # /audio/generate is non-streaming and holds the model for the whole request: unregistered, a + # non-forced swap counted zero and could tear it down mid-TTS, and a forced one had no entry. + _route_gate() + import asyncio + from types import SimpleNamespace + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + seen = {} + + class _TtsBackend: + active_model_name = "org/TTS" + models = {"org/TTS": {"is_audio": True}} + + def generate_audio_response(self, **kwargs): + # Sampled mid-generation: the window a concurrent swap would tear down in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + return (b"RIFFfake", 24000) + + # is_loaded False picks the transformers TTS branch, not the GGUF one. + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace(is_loaded = False, _is_audio = False), + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: _TtsBackend()) + + async def _no_auto_switch(*a, **k): + return None + + monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch) + + payload = ChatCompletionRequest( + model = "org/TTS", + messages = [{"role": "user", "content": "hi"}], + thread_id = "thread-tts", + ) + asyncio.run(inf_mod.generate_audio(payload, request = None, current_subject = "tester")) + + assert seen["count"] == 1 + # Named, so the swap dialog can say which chat it would interrupt. + assert seen["snapshot"][0]["thread_id"] == "thread-tts" + # And it unregisters, or one TTS call would 409 every later reload. + assert active_generations.count() == 0 + + +class _ChatRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/chat/completions reads.""" + + def __init__(self): + from types import SimpleNamespace + + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/chat/completions") + self.state = SimpleNamespace(skip_api_monitor = True) + self.scope: dict = {} + + +def _standard_chat_stubs(monkeypatch, backend): + """Point /v1/chat/completions at a standard (non-GGUF) backend. + + ``supports_tools`` False keeps the request off the safetensors server-tool + loop, which registers on its own, so the plain default branch is exercised. + """ + from types import SimpleNamespace + + import routes.inference as inf_mod + + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace( + is_loaded = False, + supports_tools = False, + is_vision = False, + context_length = None, + ), + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + monkeypatch.setattr( + inf_mod, "_detect_safetensors_features", lambda *a, **k: {"supports_tools": False} + ) + + async def _no_auto_switch(*a, **k): + return None + + monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch) + return inf_mod + + +def test_standard_non_stream_chat_is_visible_to_the_swap_gate(monkeypatch): + # ``stream`` defaults to false, so this is the default shape of a standard chat and it holds the + # worker throughout. Only the streaming branch registered, so a swap truncated the completion. + _route_gate() + import asyncio + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + seen = {} + + class _StandardBackend: + active_model_name = "org/M" + models = {"org/M": {"chat_template_info": {"template": "chatml"}}} + + def generate_chat_response( + self, + *, + cancel_event = None, + stats_holder = None, + **kwargs, + ): + # Sampled mid-generation: exactly the window an /unload lands in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + # And the gate must reach this run, on the event the decode watches. + seen["cancelled"] = active_generations.cancel_all() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield "33" + + def reset_generation_state(self, caller_cancel_event = None): + pass + + _standard_chat_stubs(monkeypatch, _StandardBackend()) + + payload = ChatCompletionRequest( + model = "org/M", + messages = [{"role": "user", "content": "hi"}], + thread_id = "thread-chat", + ) + response = asyncio.run( + inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + # Named, so the swap dialog can say which chat it would interrupt. + assert seen["snapshot"][0]["thread_id"] == "thread-chat" + assert seen["cancelled"] == 1 + assert seen["reached_the_decode"] + # And it unregisters, or one completion would 409 every later reload. + assert active_generations.count() == 0 + + +def test_standard_non_stream_chat_unregisters_when_it_fails(monkeypatch): + # A raising backend must not strand an entry: that would 409 every later swap. + _route_gate() + import asyncio + + from fastapi import HTTPException + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + class _BrokenBackend: + active_model_name = "org/M" + models = {"org/M": {"chat_template_info": {"template": "chatml"}}} + + def generate_chat_response(self, **kwargs): + raise RuntimeError("decode exploded") + yield # pragma: no cover - generator marker + + def reset_generation_state(self, caller_cancel_event = None): + pass + + _standard_chat_stubs(monkeypatch, _BrokenBackend()) + + payload = ChatCompletionRequest(model = "org/M", messages = [{"role": "user", "content": "hi"}]) + with pytest.raises(HTTPException): + asyncio.run( + inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester") + ) + + assert active_generations.count() == 0 + + +def test_audio_input_non_stream_chat_is_visible_to_the_swap_gate(monkeypatch): + # An audio-input model with the default stream=false holds the standard worker throughout. Only + # the streaming sibling registered, so a non-forced swap could unload it mid-transcription. + _route_gate() + import asyncio + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + seen = {} + + class _AudioInputBackend: + active_model_name = "org/AUDIO-IN" + models = {"org/AUDIO-IN": {"has_audio_input": True}} + + def generate_audio_input_response( + self, + *, + cancel_event = None, + **kwargs, + ): + # Sampled mid-transcription: the window a concurrent swap lands in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield "33" + + def reset_generation_state(self, caller_cancel_event = None): + pass + + _standard_chat_stubs(monkeypatch, _AudioInputBackend()) + monkeypatch.setattr(inf_mod, "_decode_audio_base64", lambda _b64: object()) + + payload = ChatCompletionRequest( + model = "org/AUDIO-IN", + messages = [{"role": "user", "content": "transcribe this"}], + audio_base64 = "ZmFrZQ==", + thread_id = "thread-audio-in", + ) + response = asyncio.run( + inf_mod.openai_chat_completions(payload, _ChatRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + assert seen["snapshot"][0]["thread_id"] == "thread-audio-in" + assert seen["cancelled"] == 1 + assert seen["reached_the_decode"] + # And it unregisters, or one transcription would 409 every later reload. + assert active_generations.count() == 0 + + +def _anthropic_route_stubs(monkeypatch, **overrides): + """Minimal GGUF backend + request stub for the /v1/messages route.""" + from types import SimpleNamespace + + import routes.inference as inf_mod + from state.tool_policy import reset_tool_policy + + reset_tool_policy() + backend = SimpleNamespace( + is_loaded = True, + is_vision = False, + supports_tools = True, + supports_tool_passthrough = True, + model_identifier = "org/M-GGUF", + base_url = "http://llama.test", + context_length = 4096, + count_chat_tokens = lambda *a, **k: 2, + ) + backend.__dict__.update(overrides) + monkeypatch.setattr(inf_mod, "get_llama_cpp_backend", lambda: backend) + monkeypatch.setattr(inf_mod, "_automatic_model_load_may_run", lambda: False) + return inf_mod + + +class _MessagesRequest(_NeverDisconnectedRequest): + """Minimal stand-in for the Starlette Request /v1/messages reads.""" + + def __init__(self): + from types import SimpleNamespace + + self.method = "POST" + self.url = SimpleNamespace(path = "/v1/messages") + self.state = SimpleNamespace(skip_api_monitor = True) + + +@pytest.mark.parametrize("with_server_tools", [False, True]) +def test_local_anthropic_non_stream_is_visible_to_the_swap_gate(monkeypatch, with_server_tools): + # ``stream`` defaults to false on /v1/messages, so the non-streaming plain and server-tool branches + # are the common shape and decode throughout. Only their streaming siblings registered. + _route_gate() + import asyncio + + from models.inference import AnthropicMessagesRequest + + seen = {} + + def _sample(): + # Sampled mid-generation: exactly the window an /unload lands in. + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + + def _gen_plain(*, cancel_event = None, **kwargs): + _sample() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield "ok" + + def _gen_tools(*, cancel_event = None, **kwargs): + _sample() + seen["reached_the_decode"] = cancel_event is not None and cancel_event.is_set() + yield {"type": "content", "text": "ok"} + + inf_mod = _anthropic_route_stubs( + monkeypatch, + generate_chat_completion = _gen_plain, + generate_chat_completion_with_tools = _gen_tools, + ) + + fields = {"max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} + if with_server_tools: + fields["enable_tools"] = True + fields["tools"] = [{"type": "web_search_20250305", "name": "web_search"}] + payload = AnthropicMessagesRequest(**fields) + + response = asyncio.run( + inf_mod.anthropic_messages(payload, request = _MessagesRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # The event registered is the one the decode watches, so a forced swap lands. + assert seen["reached_the_decode"] + # And it unregisters, or one message would 409 every later reload. + assert active_generations.count() == 0 + + +def test_anthropic_passthrough_non_stream_is_visible_to_the_swap_gate(monkeypatch): + # The client-tool pass-through holds llama-server for one non-streaming POST. Its streaming sibling + # registers inside the body generator; this branch had none, so /unload tore the server down. + _route_gate() + import asyncio + + import httpx + + from models.inference import AnthropicMessagesRequest + + seen = {} + + def handler(request): + seen["count"] = active_generations.count() + seen["snapshot"] = active_generations.snapshot() + seen["cancelled"] = active_generations.cancel_all() + return httpx.Response( + 200, + json = { + "choices": [ + {"message": {"role": "assistant", "content": "33"}, "finish_reason": "stop"} + ] + }, + ) + + inf_mod = _anthropic_route_stubs(monkeypatch) + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + # The pass-through takes a per-request client, so a Stop or forced swap can close it mid-POST. + monkeypatch.setattr( + inf_mod, + "_cancelable_nonstreaming_client", + lambda: real_async_client(transport = transport), + ) + + # enable_tools False keeps the server-tool loop out, so the client tool takes the pass-through. + payload = AnthropicMessagesRequest( + max_tokens = 16, + messages = [{"role": "user", "content": "hi"}], + enable_tools = False, + tools = [{"name": "lookup", "input_schema": {"type": "object", "properties": {}}}], + ) + + response = asyncio.run( + inf_mod.anthropic_messages(payload, request = _MessagesRequest(), current_subject = "tester") + ) + + assert response.status_code == 200 + assert seen["count"] == 1 + assert seen["snapshot"][0]["model"] == "org/M-GGUF" + assert seen["cancelled"] == 1 + # And it unregisters, or one message would 409 every later reload. + assert active_generations.count() == 0 + + +def test_anthropic_passthrough_non_stream_stops_when_the_swap_cancels_it(monkeypatch): + # Registering is half the job: a pooled client cannot be closed, so the run was cancelled while the + # POST carried on. The watcher closes a per-request client; the set event makes that error a cancel. + _route_gate() + import asyncio + + import httpx + + from models.inference import AnthropicMessagesRequest + + seen = {} + + def handler(request): + # Stand in for a forced swap mid-decode: cancel, then fail the transport as closing would. + seen["cancelled"] = active_generations.cancel_all() + raise httpx.ConnectError("client closed") + + inf_mod = _anthropic_route_stubs(monkeypatch) + transport = httpx.MockTransport(handler) + real_async_client = httpx.AsyncClient + monkeypatch.setattr( + inf_mod, + "_cancelable_nonstreaming_client", + lambda: real_async_client(transport = transport), + ) + + payload = AnthropicMessagesRequest( + max_tokens = 16, + messages = [{"role": "user", "content": "hi"}], + enable_tools = False, + tools = [{"name": "lookup", "input_schema": {"type": "object", "properties": {}}}], + ) + + with pytest.raises(asyncio.CancelledError): + asyncio.run( + inf_mod.anthropic_messages( + payload, request = _MessagesRequest(), current_subject = "tester" + ) + ) + + assert seen["cancelled"] == 1 + # Cancelled or not, the entry must go, or one message 409s every later reload. + assert active_generations.count() == 0 + + +def test_audio_generation_unregisters_when_it_fails(monkeypatch): + # A raising backend must not strand an entry: that would 409 every later load. + _route_gate() + import asyncio + from types import SimpleNamespace + + from fastapi import HTTPException + + import routes.inference as inf_mod + from models.inference import ChatCompletionRequest + + class _BrokenTtsBackend: + active_model_name = "org/TTS" + models = {"org/TTS": {"is_audio": True}} + + def generate_audio_response(self, **kwargs): + raise RuntimeError("codec exploded") + + monkeypatch.setattr( + inf_mod, + "get_llama_cpp_backend", + lambda: SimpleNamespace(is_loaded = False, _is_audio = False), + ) + monkeypatch.setattr(inf_mod, "get_inference_backend", lambda: _BrokenTtsBackend()) + + async def _no_auto_switch(*a, **k): + return None + + monkeypatch.setattr(inf_mod, "_maybe_auto_switch_model", _no_auto_switch) + + payload = ChatCompletionRequest( + model = "org/TTS", + messages = [{"role": "user", "content": "hi"}], + ) + with pytest.raises(HTTPException): + asyncio.run(inf_mod.generate_audio(payload, request = None, current_subject = "tester")) + + assert active_generations.count() == 0 + + +# ── sidecar install: carrying a confirmed swap through ───────────────── + + +def _stub_install_route(monkeypatch, *, in_flight_events): + """Point POST /install-latest-transformers at an in-memory sidecar install. + + ``in_flight_events`` stands in for the middleware's in-flight count: a + request is counted until its stream observes the cancel event and unwinds, + which is the coupling the installer's guard actually reads. + """ + from types import SimpleNamespace + + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + import utils.transformers_latest as latest_mod + import utils.transformers_version as version_mod + + calls = {"installed": [], "released": 0} + + monkeypatch.setattr(version_mod, "try_begin_sidecar_swap", lambda: True) + + def _end_sidecar_swap(): + calls["released"] += 1 + + monkeypatch.setattr(version_mod, "end_sidecar_swap", _end_sidecar_swap) + + import core.export as export_mod + import core.training as training_mod + + monkeypatch.setattr( + training_mod, + "get_training_backend", + lambda: SimpleNamespace(is_training_active = lambda: False), + ) + monkeypatch.setattr( + export_mod, + "get_export_backend", + lambda: SimpleNamespace(is_export_active = lambda: False, current_checkpoint = None), + ) + monkeypatch.setattr( + inf_mod, + "get_inference_backend", + lambda: SimpleNamespace(active_model_name = None, load_generation = 0), + ) + + def _fake_in_flight(current_request_counted = True, *, include_pending = True): + return sum(1 for ev in in_flight_events if not ev.is_set()) + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _fake_in_flight) + + def _install(version, before_swap, *args, **kwargs): + calls["installed"].append(version) + return {"success": True, "version": version, "message": "installed"} + + monkeypatch.setattr(latest_mod, "install_latest_transformers", _install) + return inf_mod, calls + + +def test_confirmed_install_stops_the_chats_it_was_given_permission_to_stop(monkeypatch): + # The install sits between the swap's "stop N chats" prompt and the /load carrying the + # confirmation, and refuses while those chats run, so a confirmed install cancels them itself. + _route_gate() + import asyncio + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev]) + + with active_generations.ActiveGeneration(ev, thread_id = "t1", model = "org/M-GGUF"): + response = asyncio.run( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True), + "tester", + ) + ) + assert ev.is_set() + + assert response.success is True + assert calls["installed"] == ["5.0.0"] + + +def test_unconfirmed_install_still_refuses_while_chats_stream(monkeypatch): + # Unchanged for every caller that never confirmed (second tab, desktop, curl): no flag, no cancel. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev]) + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0"), + "tester", + ) + ) + assert not ev.is_set() + assert active_generations.count() == 1 + + assert exc.value.status_code == 409 + assert calls["installed"] == [] + + +def test_a_confirmed_install_that_cannot_drain_refuses_instead_of_swapping(monkeypatch): + # A cancelled request that never observes its event keeps the in-flight count up, so the drain is + # bounded and cannot wedge the process holding the gate; the recheck behind it still refuses. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + stuck = threading.Event() + stuck.set() # already "cancelled", yet still counted: it never unwinds + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev, stuck]) + monkeypatch.setattr(inf_mod, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.05) + + def _never_unwinds(current_request_counted = True, *, include_pending = True): + return 1 + + import core.inference.llama_keepwarm as keepwarm + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _never_unwinds) + + async def _install(): + # Deadline here too: a regression that drops the drain's bound must fail, not hang the suite. + return await asyncio.wait_for( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True), + "tester", + ), + timeout = 5, + ) + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run(_install()) + + assert exc.value.status_code == 409 + assert calls["installed"] == [] + + +def test_confirmed_install_does_not_spend_its_cancel_on_an_install_that_will_refuse(monkeypatch): + # An unrelated counted request the cancel cannot stop must be waited out BEFORE the cancel: the + # recheck refuses while it is there, so cancelling first stopped chats for a doomed install. + _route_gate() + import asyncio + + from fastapi import HTTPException + + from models.inference import InstallLatestTransformersRequest + + ev = threading.Event() + inf_mod, calls = _stub_install_route(monkeypatch, in_flight_events = [ev]) + + import core.inference.llama_keepwarm as keepwarm + + def _never_drains(current_request_counted = True, *, include_pending = True): + # Discounting the registered chat still leaves the counted-only stranger: the drain must not clear. + return 2 + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _never_drains) + monkeypatch.setattr(inf_mod, "_POST_CANCEL_DRAIN_TIMEOUT_S", 0.05) + + async def _install(): + return await asyncio.wait_for( + inf_mod.install_latest_transformers_route( + InstallLatestTransformersRequest(version = "5.0.0", force_cancel_active = True), + "tester", + ), + timeout = 5, + ) + + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + with pytest.raises(HTTPException) as exc: + asyncio.run(_install()) + # The refusal is the same as before; what changed is that the chat lives. + assert not ev.is_set() + assert active_generations.count() == 1 + + assert exc.value.status_code == 409 + assert calls["installed"] == [] + + +# ── draining before teardown ────────────────────────────────────────── + + +def _drain_with_counts(monkeypatch, counts, **kwargs): + """Run _wait_for_model_switch_idle against a scripted in-flight count. + + ``counts`` is consumed one entry per poll; the last value repeats, so a + trailing non-zero stands for a request that never unwinds. + """ + _route_gate() + import asyncio + + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + remaining = list(counts) + polls = {"n": 0} + + def _count(current_request_counted = True, *, include_pending = True): + polls["n"] += 1 + return remaining.pop(0) if len(remaining) > 1 else remaining[0] + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0) + + async def _run(): + # Hard test-side deadline: a drain that regresses to waiting forever must fail red, not hang. + await asyncio.wait_for( + inf_mod._wait_for_model_switch_idle(current_request_counted = False, **kwargs), + timeout = 5, + ) + + asyncio.run(_run()) + return polls["n"] + + +def test_forced_swap_does_not_wait_out_the_generations_it_is_about_to_cancel(monkeypatch): + # cancel_pending discounts the registered generations, since the caller cancels them right after. + # Drop the discount and the drain waits on a count only that pending cancel can lower: forever. + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + polls = _drain_with_counts(monkeypatch, [1], cancel_pending = True) + assert polls == 1 + + +def test_the_same_drain_without_the_discount_would_keep_waiting(monkeypatch): + # The other half: that count really does block, so the previous test passes by the discount. + ev = threading.Event() + with active_generations.ActiveGeneration(ev, thread_id = "t1"): + polls = _drain_with_counts(monkeypatch, [1], timeout_s = 0.05) + assert polls > 1 + + +def test_post_cancel_drain_gives_up_on_a_request_that_never_unwinds(monkeypatch): + # TTS on the subprocess backend observes no cancel event, so a forced swap can cancel it and still + # see it counted forever. The post-cancel drains hold the gate, so they must expire and proceed. + polls = _drain_with_counts(monkeypatch, [1], timeout_s = 0.05) + assert polls > 1 + + +def test_drain_returns_as_soon_as_the_cancelled_requests_unwind(monkeypatch): + # The bound is a backstop: once the count drops the drain returns without sitting out the timeout. + polls = _drain_with_counts(monkeypatch, [2, 1, 0], timeout_s = 30) + assert polls == 3 + + +# ── queued chats must not cancel the running one ────────────────────── + + +def _orchestrator_for_ownership(): + """A real InferenceOrchestrator with just enough stubbed to drive the lock.""" + _route_gate() + orch_mod = pytest.importorskip( + "core.inference.orchestrator", reason = "inference stack not installed" + ) + orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator) + orch._gen_lock = threading.Lock() + orch._active_cancel_events = [] + orch._executing_cancel_events = [] + orch._active_cancel_lock = threading.Lock() + orch._cancel_event = threading.Event() + orch._ensure_subprocess_alive = lambda: False # stop before _send_cmd + return orch + + +def test_a_queued_chat_cannot_reset_the_chat_that_is_generating(): + # Safetensors generation serialises on _gen_lock and the worker has ONE cancel event: stopping + # queued chat B reset that shared event and killed running chat A. Scope the reset to the holder. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + + orch._claim_worker(a_event) # A holds the lock ... + orch._mark_worker_started(a_event) # ... and the worker is answering it + orch.reset_generation_state(b_event) # B is queued and gets stopped + assert not orch._cancel_event.is_set() + + orch.reset_generation_state(a_event) # A's own Stop still works + assert orch._cancel_event.is_set() + + +def test_a_global_reset_still_cancels_whatever_is_running(): + # Unload and switch pass nothing: they mean stop everything, else a generation survives teardown. + orch = _orchestrator_for_ownership() + _running = threading.Event() + orch._claim_worker(_running) + orch._mark_worker_started(_running) + orch.reset_generation_state() + assert orch._cancel_event.is_set() + + +def test_a_reset_with_no_generation_running_is_not_dropped(): + # Nothing holds the lock, so no chat to protect: a reset before any generation must still run. + orch = _orchestrator_for_ownership() + orch.reset_generation_state(threading.Event()) + assert orch._cancel_event.is_set() + + +def test_unload_waits_for_a_request_that_is_admitted_but_not_yet_registered(monkeypatch): + # The window between the keep-warm middleware and _TrackedCancel: counted in-flight, absent from + # the registry. Cancelling on the registry alone tore the backend down under an admitted request. + _route_gate() + import core.inference.llama_keepwarm as keepwarm + import routes.inference as inf_mod + + # Counted for two polls, then the request registers/finishes and clears. + remaining = [1, 1, 0] + seen = {} + + def _count(current_request_counted = True, *, include_pending = True): + return remaining.pop(0) if len(remaining) > 1 else remaining[0] + + monkeypatch.setattr(keepwarm, "other_inference_request_count", _count) + monkeypatch.setattr(inf_mod, "_switch_waiter_count", lambda: 0) + + torn_down: list[str] = [] + + def _record_teardown(): + seen["counted_at_teardown"] = remaining[0] + torn_down.append("gguf") + + # Registry deliberately empty: this is the unregistered case. + response = _run_unload( + inf_mod, + monkeypatch, + loaded_gguf = "org/A-GGUF", + requested = "org/A-GGUF", + force = True, + torn_down = torn_down, + unload_model = _record_teardown, + ) + + assert active_generations.count() == 0 + assert torn_down == ["gguf"] + assert seen["counted_at_teardown"] == 0 + assert response.status == "unloaded" + + +def test_a_dispatched_chat_cannot_reset_its_concurrently_dispatched_sibling(): + # Compare-mode / dispatched runs bypass _gen_lock and run concurrently, so with several claimed + # at once a Stop on one must still leave the others alone. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + c_event = threading.Event() + + orch._claim_worker(a_event) + orch._mark_worker_started(a_event) + orch._claim_worker(b_event) + orch._mark_worker_started(b_event) + + orch.reset_generation_state(c_event) # a third, unrelated request + assert not orch._cancel_event.is_set() + + orch.reset_generation_state(b_event) # one of the running pair + assert orch._cancel_event.is_set() + + +def test_releasing_one_generation_leaves_the_other_claimed(): + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + orch._claim_worker(a_event) + orch._mark_worker_started(a_event) + orch._claim_worker(b_event) + orch._mark_worker_started(b_event) + orch._release_worker(a_event) + + orch.reset_generation_state(a_event) # now a stranger + assert not orch._cancel_event.is_set() + + orch._release_worker(b_event) + orch.reset_generation_state(a_event) # nothing running: no one to protect + assert orch._cancel_event.is_set() + + +def test_a_dispatched_request_queued_behind_another_is_not_an_owner(): + # The subprocess runs generations one at a time, so admission is not execution: B can be claimed + # while the worker answers A. Counting B as an owner let its Stop signal the shared event and end A. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + + orch._claim_worker(a_event) + orch._mark_worker_started(a_event) # the worker answered A + orch._claim_worker(b_event) # B is only queued behind it + + orch.reset_generation_state(b_event) + assert not orch._cancel_event.is_set(), "a queued request must not reset A" + + orch._mark_worker_started(b_event) # the worker moves on to B + orch.reset_generation_state(b_event) + assert orch._cancel_event.is_set() + + +def test_a_queued_request_cannot_reset_during_the_other_ones_prefill(): + # Between _send_cmd and the first response A is claimed but not executing; treating that as + # "nobody to protect" let a queued request's Stop kill A mid-prefill. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + + orch._claim_worker(a_event) # A sent its command and is in prefill + orch._claim_worker(b_event) # B is queued behind it + + orch.reset_generation_state(b_event) + assert not orch._cancel_event.is_set(), "B must not reset A during prefill" + + # A's own Stop still works before any token has arrived. + orch.reset_generation_state(a_event) + assert orch._cancel_event.is_set() + + +def test_the_oldest_claim_is_the_one_the_worker_is_prefilling(): + # The command queue is FIFO, so with nothing answering the oldest claim is the executor. + orch = _orchestrator_for_ownership() + a_event = threading.Event() + b_event = threading.Event() + orch._claim_worker(a_event) + orch._claim_worker(b_event) + orch._release_worker(a_event) + + orch.reset_generation_state(b_event) + assert orch._cancel_event.is_set(), "B is now the oldest claim" + + +def test_claim_order_matches_send_order_under_concurrent_dispatch(): + # _owns_worker reads claim order to decide who is prefilling, so a claim not atomic with the + # enqueue can put A first in the list while B is first in the subprocess queue: stopping A kills B. + _route_gate() + orch_mod = pytest.importorskip( + "core.inference.orchestrator", reason = "inference stack not installed" + ) + orch = orch_mod.InferenceOrchestrator.__new__(orch_mod.InferenceOrchestrator) + orch._active_cancel_events = [] + orch._executing_cancel_events = [] + orch._active_cancel_lock = threading.Lock() + orch._send_order_lock = threading.Lock() + + sent: list = [] + barrier = threading.Barrier(4) + + def worker(ev): + barrier.wait(timeout = 10) + with orch._send_order_lock: + orch._claim_worker(ev) + # Stand in for _send_cmd: the enqueue must not be separable from the claim. + sent.append(ev) + + events = [threading.Event() for _ in range(4)] + threads = [threading.Thread(target = worker, args = (e,)) for e in events] + for t in threads: + t.start() + for t in threads: + t.join(timeout = 30) + + assert orch._active_cancel_events == sent, "claim order must equal send order" diff --git a/studio/backend/tests/test_anthropic_admission.py b/studio/backend/tests/test_anthropic_admission.py index de01accd08..d4fcf85a45 100644 --- a/studio/backend/tests/test_anthropic_admission.py +++ b/studio/backend/tests/test_anthropic_admission.py @@ -641,12 +641,13 @@ def test_every_dispatch_site_goes_through_admission(): for node in ast.walk(tree) if isinstance(node, ast.AsyncFunctionDef) and node.name == "anthropic_messages" ) - # The wrappers themselves call _monitored_anthropic; only the dispatch sites count. + # The wrappers themselves call _monitored_anthropic (the non-streaming one + # through the swap-gate tracker); only the dispatch sites count. nested = { node for node in ast.walk(handler) if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) - and node.name.startswith("_admitted_anthropic") + and node.name.startswith(("_admitted_anthropic", "_tracked_anthropic")) } inner = {id(n) for wrapper in nested for n in ast.walk(wrapper)} @@ -763,12 +764,13 @@ def _passthrough_payload(**fields): return _payload(tools = _CLIENT_TOOLS, enable_tools = False, **fields) -def test_response_pre_start_cleanup_exits_the_passthrough_tracker(monkeypatch): - """A disconnect before the body starts must still exit the cancel tracker. +def test_response_pre_start_cleanup_leaves_no_passthrough_tracker(monkeypatch): + """A disconnect before the body starts must leave no tracker and no slot. - The wrapper replaces the response's own pre-start hook, so it has to chain to - it. Asserting through _CANCEL_REGISTRY rather than the wiring, because the - hook can be present and still be a no-op. + The passthrough registers from inside its body rather than eagerly, so a + generator that never runs registers nothing; the hook still has to hand the + admission slot back. Asserting through _CANCEL_REGISTRY and the pool rather + than the wiring, because the hook can be present and still be a no-op. """ backend = _install_backend(monkeypatch, slots = 1) backend.supports_tool_passthrough = True @@ -778,7 +780,7 @@ def test_response_pre_start_cleanup_exits_the_passthrough_tracker(monkeypatch): response = await anthropic_messages( _passthrough_payload(stream = True), request = _Request(), current_subject = "t" ) - assert inf_mod._CANCEL_REGISTRY, "passthrough should have registered a tracker" + assert inf_mod._CANCEL_REGISTRY == {}, "nothing runs the body's exit for it yet" cleanup = getattr(response, "_unstarted_cleanup", None) assert cleanup is not None diff --git a/studio/backend/tests/test_anthropic_passthrough_respawn.py b/studio/backend/tests/test_anthropic_passthrough_respawn.py index a9f31208ed..daa30e39c2 100644 --- a/studio/backend/tests/test_anthropic_passthrough_respawn.py +++ b/studio/backend/tests/test_anthropic_passthrough_respawn.py @@ -74,6 +74,10 @@ class _Request: class _FakeNonStreamingClient: def __init__(self): self.urls = [] + self.closed = False + + async def aclose(self): + self.closed = True async def post(self, url, **_kwargs): self.urls.append(url) @@ -189,7 +193,7 @@ def test_retry_url_tolerates_a_backend_without_respawn_hooks(): def test_non_streaming_retries_against_the_new_port(monkeypatch): client = _FakeNonStreamingClient() - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) backend = _Backend() response = asyncio.run(_run_non_streaming(backend)) @@ -201,7 +205,7 @@ def test_non_streaming_retries_against_the_new_port(monkeypatch): def test_non_streaming_raises_when_the_server_stays_dead(monkeypatch): client = _FakeNonStreamingClient() - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) backend = _Backend(respawn_ok = False) with pytest.raises(httpx.ConnectError): @@ -212,7 +216,7 @@ def test_non_streaming_raises_when_the_server_stays_dead(monkeypatch): def test_non_streaming_does_not_retry_an_mtp_crash(monkeypatch): client = _FakeNonStreamingClient() - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) backend = _Backend(mtp_handled = True) with pytest.raises(httpx.ConnectError): diff --git a/studio/backend/tests/test_inference_dispatcher_resilience.py b/studio/backend/tests/test_inference_dispatcher_resilience.py index 6184496d78..ea903a6ce0 100644 --- a/studio/backend/tests/test_inference_dispatcher_resilience.py +++ b/studio/backend/tests/test_inference_dispatcher_resilience.py @@ -39,6 +39,7 @@ def _dispatcher(): o._dispatcher_stop = threading.Event() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} return o @@ -118,3 +119,68 @@ def test_route_llama_streaming_async_clients_disable_proxy_env(): kw.arg == "trust_env" and isinstance(kw.value, ast.Constant) and kw.value.value is False for kw in call.keywords ), f"httpx.AsyncClient at line {call.lineno} must set trust_env=False" + + +def _direct_reader_host(): + """Orchestrator with only what _direct_reader and the ownership helpers touch.""" + o = InferenceOrchestrator.__new__(InferenceOrchestrator) + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + o._request_cancel_events = {} + o._active_cancel_lock = threading.Lock() + o._active_cancel_events = [] + o._executing_cancel_events = [] + o._dispatcher_thread = None + return o + + +def test_rerouting_a_foreign_response_moves_worker_ownership(): + # A _gen_lock reader already blocked on resp_queue can beat the compare dispatcher to + # that request's first response. The compare consumer passes mark_started=False, so if + # this path does not promote it nothing does: the direct request stays recorded as the + # executor, so the compare chat's Stop is ignored and a late reset from the direct one + # cancels the compare generation instead. + o = _direct_reader_host() + mine, theirs = threading.Event(), threading.Event() + o._request_cancel_events = {"mine": mine, "theirs": theirs} + o._claim_worker(mine) + o._mark_worker_started(mine) + o._claim_worker(theirs) + compare_mailbox = queue.Queue() + o._mailboxes["theirs"] = compare_mailbox + + read_one, _drain, release = _direct_reader_calls(o, "mine") + o._scripted = [{"request_id": "theirs", "type": "token", "text": "hi"}] + + assert read_one(timeout = 0.1) is None, "a foreign response is routed, not returned" + assert compare_mailbox.get_nowait()["text"] == "hi" + assert o._owns_worker(theirs), "the compare request is the one the worker answered" + assert not o._owns_worker(mine), "so a late reset from the direct request must not fire" + release() + + +def test_rerouting_a_foreign_gen_done_retires_that_request(): + # The other half of the dispatcher's move: once its last response is routed, the + # request no longer owns the worker, or a Stop for it would end whatever starts next. + o = _direct_reader_host() + mine, theirs = threading.Event(), threading.Event() + o._request_cancel_events = {"mine": mine, "theirs": theirs} + o._claim_worker(theirs) + o._mark_worker_started(theirs) + o._claim_worker(mine) + o._mailboxes["theirs"] = queue.Queue() + + read_one, _drain, release = _direct_reader_calls(o, "mine") + o._scripted = [{"request_id": "theirs", "type": "gen_done"}] + + assert read_one(timeout = 0.1) is None + assert not o._owns_worker(theirs), "retired once its last response was routed" + assert o._owns_worker(mine), "the next claim takes over" + release() + + +def _direct_reader_calls(o, request_id): + """_direct_reader wired to a scripted _read_resp (o._scripted, popped in order).""" + o._read_resp = lambda timeout = 1.0: o._scripted.pop(0) if o._scripted else None + return o._direct_reader(request_id) diff --git a/studio/backend/tests/test_llama_admission.py b/studio/backend/tests/test_llama_admission.py index f69eb7c5c9..9ff19ec27d 100644 --- a/studio/backend/tests/test_llama_admission.py +++ b/studio/backend/tests/test_llama_admission.py @@ -847,3 +847,222 @@ def test_dead_waiters_stop_counting_against_the_queue_limit(): assert queue.is_idle() asyncio.run(_run()) + + +def test_parking_frees_the_slot_for_a_waiter(): + """A holder waiting on a tool approval must not hold a decode slot. + + It is not generating, and with several prompts unanswered every slot would + be held by a run parked on a human while llama-server sits idle. + """ + + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + first = queue.reserve(capacity = 1, config = config) + second = queue.reserve(capacity = 1, config = config) + first_lease = first.lease_nowait() + assert first_lease is not None + assert second.lease_nowait() is None + + first_lease.park() + assert first_lease.slot is None, "the slot went back to the pool" + second_lease = await second.wait(0.1) + assert second_lease is not None, "parking did not free the slot" + + # The parked holder keeps its lease, so releasing it is still correct. + first_lease.unpark() + first_lease.release() + second_lease.release() + assert queue.snapshot().active == 0 + + asyncio.run(_run()) + + +def test_unpark_without_park_is_a_no_op(): + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + first = queue.reserve(capacity = 1, config = config) + first_lease = first.lease_nowait() + assert first_lease is not None + first_lease.unpark() + first_lease.unpark() + + second = queue.reserve(capacity = 1, config = config) + assert second.lease_nowait() is None, "capacity leaked past the limit" + + asyncio.run(_run()) + + +def test_releasing_a_parked_lease_leaves_the_queue_evictable(): + # is_idle() drives registry eviction, and a parked holder owns no slot, so + # nothing but the parked count keeps its queue alive. A stuck count would + # pin every dead queue for the life of the process. + async def _run(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + lease = queue.reserve(capacity = 1, config = config).lease_nowait() + lease.park() + assert not queue.is_idle(), "a parked holder is coming back to this queue" + lease.release() + assert queue.is_idle() + + asyncio.run(_run()) + + +def test_unpark_waits_instead_of_putting_two_holders_on_one_slot(): + # park() hands the freed slot to a waiter, so by the time the user answers an approval + # prompt someone else may be decoding in it. Resuming regardless left two holders + # against capacity 1, and the resumed tool loop went past the admission limit. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None, "A takes the only slot" + b = queue.reserve(capacity = 1, config = config) + assert b.lease_nowait() is None, "B waits behind A" + + a_lease.park() # A parks on an approval prompt; its slot goes to B + b_lease = await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) + assert b_lease is not None, "B was granted the parked slot" + + # A answers the prompt while B is still decoding: it must WAIT. + resumed = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.05) + assert not resumed.done(), "A must not resume while B holds the slot" + assert queue.snapshot().active <= 1, "never over capacity while waiting" + + b_lease.release() + await asyncio.wait_for(resumed, timeout = 2) + assert a_lease.slot is not None, "A took a real slot back" + assert queue.snapshot().active <= 1, "still within capacity after resuming" + + asyncio.run(scenario()) + + +def test_unpark_gives_up_when_the_caller_is_cancelled(): + # A holder being torn down must not sit in the wait loop. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + b = queue.reserve(capacity = 1, config = config) + a_lease.park() + assert await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) is not None + + ev = threading.Event() + waiting = asyncio.ensure_future(a_lease.unpark_async(cancel_event = ev, poll_s = 0.01)) + await asyncio.sleep(0.03) + assert not waiting.done() + ev.set() + await asyncio.wait_for(waiting, timeout = 2) + assert a_lease.slot is None, "gave up without a slot rather than over-admitting" + + asyncio.run(scenario()) + + +def test_an_approved_chat_is_not_overtaken_by_later_arrivals(): + # A parks on an approval prompt, B takes the slot, C arrives afterwards. release() grants + # under the same lock, so a plain poll in unpark_async never saw a free slot: A waited + # behind every later arrival and starved. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + b = queue.reserve(capacity = 1, config = config) + a_lease.park() # A's slot goes to B + b_lease = await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) + assert b_lease is not None + + # A is approved and starts waiting; C arrives only after that. + resumed = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.03) + c = queue.reserve(capacity = 1, config = config) + assert c.lease_nowait() is None + + b_lease.release() # the slot frees exactly once + await asyncio.wait_for(resumed, timeout = 2) + # A resumed; C is still queued behind it rather than having overtaken it. + assert c.lease_nowait() is None + assert queue.snapshot().active <= 1 + + asyncio.run(scenario()) + + +def test_two_approved_chats_do_not_block_each_other(): + # A bare pending-count made every approved holder count against every other: park A, admit + # and park B, admit C, approve both, and once C released the predicate stayed false forever. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + b = queue.reserve(capacity = 1, config = config) + a_lease.park() # A parks; B is admitted + b_lease = await asyncio.wait_for(b.wait(timeout_s = 1), timeout = 2) + assert b_lease is not None + + c = queue.reserve(capacity = 1, config = config) + b_lease.park() # B parks too; C is admitted + c_lease = await asyncio.wait_for(c.wait(timeout_s = 1), timeout = 2) + assert c_lease is not None + + # Both approvals come back while C is still decoding. + first = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.02) + second = asyncio.ensure_future(b_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.02) + assert not first.done() and not second.done() + + c_lease.release() + # The earlier approval goes first; the other follows once it releases. + await asyncio.wait_for(first, timeout = 2) + assert not second.done(), "the second approval waits its turn, not forever" + a_lease.release() + await asyncio.wait_for(second, timeout = 2) + assert queue.snapshot().active <= 1 + + asyncio.run(scenario()) + + +def test_an_immediate_arrival_cannot_take_an_approved_chats_slot(): + # The fairness reservation lived only in _grant_waiters_locked. reserve()'s fast path + # ignored it, so a request arriving in the window between the slot freeing and the + # approved chat's next poll took the slot straight off the top. + async def scenario(): + queue = get_llama_admission_queue("http://llama.test") + config = LlamaAdmissionConfig() + + a = queue.reserve(capacity = 1, config = config) + a_lease = a.lease_nowait() + assert a_lease is not None + a_lease.park() # A is on an approval prompt; its slot is up for grabs + b = queue.reserve(capacity = 1, config = config) + b_lease = b.lease_nowait() + assert b_lease is not None + + resumed = asyncio.ensure_future(a_lease.unpark_async(poll_s = 0.01)) + await asyncio.sleep(0.03) # A is approved and now holds a ticket + + # No await between these two: C arrives before A's poll can run again. + b_lease.release() + c = queue.reserve(capacity = 1, config = config) + assert c.lease_nowait() is None, "the freed slot is reserved for the approved chat" + + await asyncio.wait_for(resumed, timeout = 2) + assert queue.snapshot().active <= 1 + + asyncio.run(scenario()) diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py index cf41d540f1..7b20063892 100644 --- a/studio/backend/tests/test_llama_cpp_tool_loop.py +++ b/studio/backend/tests/test_llama_cpp_tool_loop.py @@ -2076,6 +2076,50 @@ def test_large_python_tool_call_emits_early_provisional_start(monkeypatch): assert any(e.get("type") == "tool_end" and e.get("tool_name") == "python" for e in events) +def test_gated_python_call_still_streams_its_arguments(monkeypatch): + """A call awaiting approval still streams its code into the card. + + Suppressing it left the chat completely blank for as long as the model took + to write the payload, which for a large file is minutes. Nothing runs before + the decision either way, and the code is what the user is approving. + """ + + big_code = "total = 0\n" + "\n".join(f"total += {i}" for i in range(120)) + assert len(json.dumps({"code": big_code})) > _PROVISIONAL_ARGS_MIN_CHARS + + first_stream = _streamed_structured_tool_call("python", {"code": big_code}, "call_gated") + final_stream = [_sse({"content": "Done."}), _done()] + payloads: list[dict] = [] + backend = _make_backend(monkeypatch, [first_stream, final_stream], payloads) + + monkeypatch.setattr("core.inference.tools.execute_tool", lambda name, arguments, **_k: "OK") + monkeypatch.setattr("core.inference.llama_cpp.wait_tool_decision", lambda *_a, **_k: "allow") + + events = list( + backend.generate_chat_completion_with_tools( + messages = [{"role": "user", "content": "write code"}], + tools = [{"type": "function", "function": {"name": "python"}}], + confirm_tool_calls = True, + permission_mode = "ask", + max_tool_iterations = 1, + ) + ) + + tool_starts = [e for e in events if e.get("type") == "tool_start"] + provisional = [e for e in tool_starts if not e.get("arguments")] + assert len(provisional) == 1, tool_starts + assert provisional[0]["tool_call_id"] == "call_gated" + + args_events = [e for e in events if e.get("type") == "tool_args"] + assert args_events, "gated call streamed no arguments" + assert "total += 119" in "".join(e["text"] for e in args_events) + + # The approval prompt still fires, and it comes after the code is on screen. + gated = [e for e in tool_starts if e.get("awaiting_confirmation")] + assert gated, tool_starts + assert events.index(provisional[0]) < events.index(gated[0]) + + def test_auto_mode_render_html_suppresses_provisional_card_under_confirm(monkeypatch): """render_html is no longer unconditionally safe (a networked canvas asks), so with confirm_tool_calls set under permission_mode="auto" its early provisional diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py index 065eddfe99..e29fc07a95 100644 --- a/studio/backend/tests/test_openai_auto_switch.py +++ b/studio/backend/tests/test_openai_auto_switch.py @@ -1606,22 +1606,28 @@ def test_load_route_holds_lifecycle_gate(monkeypatch): def test_model_replacements_recheck_sidecar_swap_before_either_backend_is_unloaded(): - # Both replacement directions drain active inference, then recheck whether a - # sidecar install reserved the lifecycle gate during that wait. Exact-model - # reuse exits earlier, so an already-loaded model never waits on unrelated inference. + # Both replacement directions drain, then recheck whether a sidecar install reserved the + # gate meanwhile. That recheck is the last thing that can reject the load, so the + # destructive cancel must follow it. Exact-model reuse exits earlier and never waits. import inspect src = inspect.getsource(inference_route._load_model_impl) + already_loaded = src.index('status = "already_loaded"') + standard_branch = src.index("# ── Standard path") + gguf_wait = src.index("await _wait_for_model_switch_idle", src.index("if config.is_gguf:")) gguf_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", gguf_wait) + gguf_cancel = src.index("on_reload_confirmed(cancel = True)", gguf_wait) unload_unsloth = src.index("unsloth_backend.unload_model", gguf_wait) - standard_wait = src.index("await _wait_for_model_switch_idle", gguf_wait + 1) - standard_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", standard_wait) - unload_gguf = src.index("llama_backend.unload_model()", standard_wait) - already_loaded = src.index('status = "already_loaded"') - assert already_loaded < gguf_wait < gguf_sidecar_check < unload_unsloth - assert standard_wait < standard_sidecar_check < unload_gguf + standard_wait = src.index("await _wait_for_model_switch_idle", standard_branch) + standard_sidecar_check = src.index("_raise_if_sidecar_swap_in_progress()", standard_wait) + standard_cancel = src.index("on_reload_confirmed(cancel = True)", standard_wait) + unload_gguf = src.index("llama_backend.unload_model()", standard_wait) + + assert already_loaded < gguf_wait < gguf_sidecar_check < gguf_cancel < unload_unsloth + assert standard_branch < standard_wait < standard_sidecar_check + assert standard_sidecar_check < standard_cancel < unload_gguf def test_switch_waiter_deregisters_before_swap_gate_release(): diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py index 7758339070..eeb6cee871 100644 --- a/studio/backend/tests/test_openai_tool_passthrough.py +++ b/studio/backend/tests/test_openai_tool_passthrough.py @@ -4615,6 +4615,9 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"prompt": "hi", "stream": False} + async def is_disconnected(self): + return False + class FailingAsyncClient: async def __aenter__(self): return self @@ -4622,14 +4625,18 @@ class TestApiMonitorProviderAndCompletionStreams: async def __aexit__(self, *_args): return False + async def aclose(self): + return None + async def post(self, *_args, **_kwargs): raise httpx.ConnectError("llama down") monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) + # Per-request client so a forced swap can close it mid-call; the pooled one is shared. monkeypatch.setattr( inf_mod, - "nonstreaming_client", + "_cancelable_nonstreaming_client", lambda: FailingAsyncClient(), ) monkeypatch.setattr( @@ -4667,9 +4674,15 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"prompt": "hi", "stream": False} + async def is_disconnected(self): + return False + captured = [] class CapturingClient: + async def aclose(self): + return None + async def post(self, _url, *, json, **_kwargs): captured.append(dict(json)) return httpx.Response( @@ -4687,7 +4700,9 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: CapturingClient()) + monkeypatch.setattr( + inf_mod, "_cancelable_nonstreaming_client", lambda: CapturingClient() + ) monkeypatch.setattr( inf_mod, "get_llama_cpp_backend", @@ -4718,9 +4733,15 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"prompt": "hi", "stream": False, "max_tokens": 0} + async def is_disconnected(self): + return False + captured = [] class CapturingClient: + async def aclose(self): + return None + async def post(self, _url, *, json, **_kwargs): captured.append(dict(json)) return httpx.Response( @@ -4738,7 +4759,9 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: CapturingClient()) + monkeypatch.setattr( + inf_mod, "_cancelable_nonstreaming_client", lambda: CapturingClient() + ) monkeypatch.setattr( inf_mod, "get_llama_cpp_backend", @@ -4776,6 +4799,7 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: UnusedClient()) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: UnusedClient()) monkeypatch.setattr( inf_mod, "get_llama_cpp_backend", @@ -4880,6 +4904,9 @@ class TestApiMonitorProviderAndCompletionStreams: async def json(self): return {"input": ["alpha", "beta"], "model": "embed"} + async def is_disconnected(self): + return False + class FakeAsyncClient: async def __aenter__(self): return self @@ -4887,6 +4914,9 @@ class TestApiMonitorProviderAndCompletionStreams: async def __aexit__(self, *_args): return False + async def aclose(self): + return None + async def post(self, *_args, **_kwargs): assert monitor.active_count() == 1 return httpx.Response( @@ -4899,9 +4929,10 @@ class TestApiMonitorProviderAndCompletionStreams: monitor = ApiMonitor(max_entries = 3) monkeypatch.setattr(inf_mod, "api_monitor", monitor) + # Per-request client so a forced swap can close it mid-call; the pooled one is shared. monkeypatch.setattr( inf_mod, - "nonstreaming_client", + "_cancelable_nonstreaming_client", lambda: FakeAsyncClient(), ) monkeypatch.setattr( @@ -6372,7 +6403,7 @@ class TestApiMonitorSafetensorsUsage: } yield "safe reply" - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): pass monitor = ApiMonitor(max_entries = 3) @@ -6443,7 +6474,7 @@ class TestApiMonitorSafetensorsUsage: cancel_event.set() yield {"type": "content", "text": "ignored"} - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): pass monitor = ApiMonitor(max_entries = 3) @@ -6504,7 +6535,7 @@ class TestApiMonitorSafetensorsUsage: def generate_chat_completion_with_tools(self, **_kwargs): yield {"type": "content", "text": "unused"} - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): nonlocal reset_called reset_called = True diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py index 3a36500aee..7963b71e8e 100644 --- a/studio/backend/tests/test_orchestrator_unload_cancel.py +++ b/studio/backend/tests/test_orchestrator_unload_cancel.py @@ -19,6 +19,10 @@ def _bare_orchestrator(): """An orchestrator without the real __init__ subprocess/network.""" o = InferenceOrchestrator.__new__(InferenceOrchestrator) o._gen_lock = threading.Lock() + o._send_order_lock = threading.Lock() + o._active_cancel_lock = threading.Lock() + o._active_cancel_events = [] + o._executing_cancel_events = [] o._cancel_event = threading.Event() # stands in for the mp.Event o._drain_event = threading.Event() # stands in for the unload-drain mp.Event o._proc = object() # truthy so _ensure_subprocess_alive reports alive @@ -775,6 +779,7 @@ def test_dispatched_bails_when_unload_flips_before_mailbox_registration(monkeypa o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) monkeypatch.setattr(o, "_start_dispatcher", lambda: None) @@ -817,6 +822,7 @@ def test_dispatched_bails_when_model_swapped_before_mailbox_registration(monkeyp o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -846,6 +852,7 @@ def test_dispatched_bails_when_dispatcher_stopped_before_mailbox_registration(mo o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -872,6 +879,7 @@ def test_dispatched_happy_path_registers_and_sends(monkeypatch): o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1338,6 +1346,7 @@ def test_dispatched_bail_stops_orphan_dispatcher_it_started(monkeypatch): o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = None # none running -> this call starts it monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1382,6 +1391,7 @@ def test_dispatched_bail_keeps_dispatcher_with_other_active_mailbox(monkeypatch) o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = None monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1419,6 +1429,7 @@ def test_dispatched_bail_keeps_preexisting_dispatcher(monkeypatch): o = _bare_orchestrator() o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._unload_pending = False o._dispatcher_thread = _AliveDispatcher() # already running monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) @@ -1545,6 +1556,7 @@ def test_concurrent_start_dispatcher_spawns_exactly_one(): o._resp_queue = _queue.Queue() # real queue so the dispatcher loop blocks and stays alive o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._dispatcher_thread = None o._dispatcher_stop = threading.Event() o._dispatcher_lifecycle_lock = threading.Lock() @@ -1660,6 +1672,7 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher(): o._resp_queue = _queue.Queue() # a spawned dispatcher would block-read here and stay alive o._mailbox_lock = threading.Lock() o._mailboxes = {} + o._request_cancel_events = {} o._dispatcher_stop = threading.Event() o._dispatcher_lifecycle_lock = threading.Lock() o._unload_pending = False @@ -1713,3 +1726,310 @@ def test_queued_start_behind_unload_stop_spawns_no_dispatcher(): assert o._dispatcher_thread is None, "the stop cleared it and the queued start spawned nothing" live = [t for t in threading.enumerate() if t.name == "inference-dispatcher" and t.is_alive()] assert live == [], "no fresh dispatcher may be left to consume the unloaded reply" + + +def _dispatch(o, resps): + """Run the dispatcher over a fixed response list and stop it.""" + import queue as _queue + + o._resp_queue = _queue.Queue() + for r in resps: + o._resp_queue.put(r) + o._dispatcher_stop = threading.Event() + t = threading.Thread(target = o._dispatcher_loop, daemon = True) + t.start() + deadline = time.monotonic() + 5.0 + while not o._resp_queue.empty() and time.monotonic() < deadline: + time.sleep(0.01) + o._dispatcher_stop.set() + t.join(timeout = 5.0) + + +def test_worker_ownership_follows_the_worker_not_the_consumer(): + # The subprocess runs one generation at a time and can start B while A's consumer has yet to + # drain its mailbox. A must stop owning the worker the moment its gen_done is routed, else + # a late Stop for A cancels B. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._mailboxes = {"a": _queue.Queue(), "b": _queue.Queue()} + o._request_cancel_events = {"a": a_cancel, "b": b_cancel} + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + + _dispatch(o, [{"type": "token", "request_id": "a", "token": "hi"}]) + assert o._owns_worker(a_cancel), "the request the worker is answering owns it" + assert not o._owns_worker(b_cancel), "a queued request does not" + + # A finishes. B has been sent but has not answered yet (it is prefilling), so the gap + # between the two is the window a late Stop for A used to fire into. + _dispatch(o, [{"type": "gen_done", "request_id": "a"}]) + assert not o._owns_worker(a_cancel), "a finished request stops owning the worker" + assert o._owns_worker(b_cancel), "the next queued request is the one prefilling" + + # Worker moves on to B, still before A's consumer reads anything. + _dispatch(o, [{"type": "token", "request_id": "b", "token": "yo"}]) + assert not o._owns_worker(a_cancel), "a finished request must not cancel its successor" + assert o._owns_worker(b_cancel), "the worker moved on to B, so B owns it" + + # A's own stream unwinding afterwards must not disturb B. + o._release_worker(a_cancel) + assert o._owns_worker(b_cancel) + + +def test_status_responses_do_not_transfer_worker_ownership(): + # Status lines are not an answer to any request; the dispatcher drops them before routing. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._mailboxes = {"a": _queue.Queue(), "b": _queue.Queue()} + o._request_cancel_events = {"a": a_cancel, "b": b_cancel} + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + + _dispatch(o, [{"type": "status", "request_id": "b", "message": "loading"}]) + # Nothing has answered, so the oldest claim is still the one prefilling. + assert o._owns_worker(a_cancel) + assert not o._owns_worker(b_cancel) + + +def test_only_the_latest_responder_executes(): + # The subprocess runs one generation at a time, so answering B means it has left A. + # _generate_inner promotes from its own consumer and can share the worker with a + # dispatched request, so the two must not both count as executing. + o = _bare_orchestrator() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + + o._mark_worker_started(a_cancel) + assert o._owns_worker(a_cancel) + o._mark_worker_started(b_cancel) + assert o._owns_worker(b_cancel), "the latest responder is the one executing" + assert not o._owns_worker(a_cancel), "and it is the only one" + # Idempotent: more of B's own tokens must not disturb it. + o._mark_worker_started(b_cancel) + assert o._owns_worker(b_cancel) + + +def test_a_stale_mailbox_read_does_not_cancel_the_running_generation(): + # A dispatched consumer can still be draining tokens after the dispatcher retired its request + # and started the next one. Stopping it then must tear down only its own stream: signalling + # the shared worker event would end its successor. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + a_cancel, b_cancel = threading.Event(), threading.Event() + o._mailboxes = {"a": _queue.Queue(), "b": _queue.Queue()} + o._request_cancel_events = {"a": a_cancel, "b": b_cancel} + o._claim_worker(a_cancel) + o._claim_worker(b_cancel) + # Worker finished A and moved on to B. + _dispatch( + o, + [ + {"type": "gen_done", "request_id": "a"}, + {"type": "token", "request_id": "b", "token": "yo"}, + ], + ) + assert o._owns_worker(b_cancel) and not o._owns_worker(a_cancel) + + # A's consumer now reads a token buffered before that, with A stopped. + a_cancel.set() + stale = [{"type": "token", "request_id": "a", "text": "late"}] + drained = [] + list( + o._consume_token_stream( + lambda timeout: stale.pop(0) if stale else None, + lambda: drained.append(True), + crash_context = "generation", + cancel_event = a_cancel, + mark_started = False, + ) + ) + assert drained, "the stopped stream still tears itself down" + assert not o._cancel_event.is_set(), "a retired request must not signal the shared worker event" + + # The generation that does own the worker still can. + b_cancel.set() + stale_b = [{"type": "token", "request_id": "b", "text": "live"}] + list( + o._consume_token_stream( + lambda timeout: stale_b.pop(0) if stale_b else None, + lambda: None, + crash_context = "generation", + cancel_event = b_cancel, + mark_started = False, + ) + ) + assert o._cancel_event.is_set(), "the running generation's own Stop must reach the worker" + + +def test_a_dispatcher_started_mid_stream_still_reaches_the_direct_reader(): + # A compare request can start the dispatcher while an ordinary chat is streaming. The + # dispatcher then owns resp_queue, and without a mailbox for the direct reader it dropped + # that chat's tokens and its gen_done as unaddressed, hanging it. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + o._request_cancel_events = {} + + read_one, _drain, release = o._direct_reader("direct-1") + try: + _dispatch( + o, + [ + {"type": "token", "request_id": "direct-1", "text": "hi"}, + {"type": "gen_done", "request_id": "direct-1"}, + ], + ) + assert read_one(timeout = 0.1) == { + "type": "token", + "request_id": "direct-1", + "text": "hi", + }, "the dispatcher must route to the direct reader, not drop" + assert read_one(timeout = 0.1)["type"] == "gen_done" + finally: + release() + assert o._direct_mailboxes == {}, "the mailbox is dropped when the stream ends" + + +def test_the_direct_reader_hands_back_a_compare_response_it_took(): + # The mirror race: this reader is already blocked on resp_queue when a compare request's + # dispatcher starts, so it can take that request's response first. Consuming it would + # corrupt this chat and hang the compare pane. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + compare_box: _queue.Queue = _queue.Queue() + o._mailboxes = {"compare-1": compare_box} + o._direct_mailboxes = {} + o._request_cancel_events = {} + o._resp_queue = _queue.Queue() + o._dispatcher_thread = None # no dispatcher yet: this reader owns the queue + + read_one, _drain, release = o._direct_reader("direct-1") + try: + o._resp_queue.put({"type": "token", "request_id": "compare-1", "text": "theirs"}) + o._resp_queue.put({"type": "token", "request_id": "direct-1", "text": "mine"}) + assert read_one(timeout = 0.1) is None, "a foreign response is not ours to yield" + assert compare_box.get_nowait()["text"] == "theirs", "it goes to its own mailbox" + assert read_one(timeout = 0.1)["text"] == "mine" + finally: + release() + + +def test_a_direct_mailbox_is_not_mistaken_for_compare_activity(): + # _mailboxes means "compare requests are in flight" to the unload and distributed paths, + # so an ordinary chat's mailbox must live somewhere else. + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + _read_one, _drain, release = o._direct_reader("direct-1") + try: + assert o._mailboxes == {} + assert "direct-1" in o._direct_mailboxes + finally: + release() + + +def test_replacing_the_subprocess_clears_worker_scoped_state(): + # Ownership is keyed only by cancel-event identity, so a consumer still blocked on its + # mailbox when the worker was replaced stayed recorded as the executor. A generation on + # the fresh worker then failed _owns_worker and could not be stopped. + import queue as _queue + + o = _bare_orchestrator() + o._mailbox_lock = threading.Lock() + dead = threading.Event() + o._mailboxes = {"compare-1": _queue.Queue()} + o._direct_mailboxes = {"direct-1": _queue.Queue()} + o._request_cancel_events = {"compare-1": dead} + o._claim_worker(dead) + o._mark_worker_started(dead) + assert o._owns_worker(dead) + + o._reset_worker_scoped_state() + + assert o._mailboxes == {} and o._direct_mailboxes == {} + assert o._request_cancel_events == {} + assert o._active_cancel_events == [] and o._executing_cancel_events == [] + # A generation on the fresh worker owns it rather than being refused by a ghost. + fresh = threading.Event() + o._claim_worker(fresh) + assert o._owns_worker(fresh), "the dead worker's request must not outrank a live one" + + +def test_audio_input_claims_the_worker_before_sending(): + # Unclaimed, a compare request queued behind an audio-input generation looked like the + # oldest owner, so stopping that queued request signalled the worker and killed this. + import ast + import pathlib + + src = pathlib.Path(orch_mod.__file__).read_text(encoding = "utf-8") + tree = ast.parse(src) + fn = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.FunctionDef) and n.name == "_generate_audio_input_inner" + ) + body = ast.get_source_segment(src, fn) or "" + claim = body.find("self._claim_worker(cancel_event)") + send = body.find("self._send_cmd(cmd)") + assert claim != -1, "_generate_audio_input_inner must claim the worker" + assert send != -1 + assert claim < send, "the claim has to happen before the command is enqueued" + assert "with self._send_order_lock:" in body, "claim and send must be one critical section" + assert "self._release_worker(cancel_event)" in body + + +def test_generation_stopped_while_queued_is_never_sent(monkeypatch): + # Two chats on the serialized backend: the second blocks on _gen_lock, and Stop sets its + # event while it waits. Sending anyway occupied the worker with a run the user ended -- + # the cancel is only checked on a token, so a long prefill (or a generation that reaches + # gen_done without one) still held up its siblings. + o = _bare_orchestrator() + monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) + monkeypatch.setattr(o, "_wait_dispatcher_idle", lambda *a, **k: None) + monkeypatch.setattr( + o, "_send_cmd", lambda cmd: pytest.fail("must not send a generation already stopped") + ) + stopped = threading.Event() + stopped.set() + + out = list( + o._generate_inner(messages = [{"role": "user", "content": "hi"}], cancel_event = stopped) + ) + + assert out == [], "a stopped request yields nothing rather than an error banner" + assert o._active_cancel_events == [], "it must not claim the worker either" + assert o._gen_lock.acquire(blocking = False) + o._gen_lock.release() + + +def test_audio_input_stopped_while_queued_is_never_sent(monkeypatch): + # Same lock, same hole. + o = _bare_orchestrator() + monkeypatch.setattr(o, "_ensure_subprocess_alive", lambda: True) + monkeypatch.setattr( + o, "_send_cmd", lambda cmd: pytest.fail("must not send a generation already stopped") + ) + stopped = threading.Event() + stopped.set() + + out = list(o._generate_audio_input_inner(audio_array = [0.0, 0.1], cancel_event = stopped)) + + assert out == [] + assert o._active_cancel_events == [] + assert o._gen_lock.acquire(blocking = False) + o._gen_lock.release() diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py index da261e8d0d..5a01839914 100644 --- a/studio/backend/tests/test_passthrough_healing.py +++ b/studio/backend/tests/test_passthrough_healing.py @@ -504,11 +504,12 @@ def _upstream_message( class ScriptedClient: - """Fake nonstreaming_client() returning scripted JSON bodies, counting POSTs.""" + """Fake upstream client returning scripted JSON bodies, counting POSTs.""" def __init__(self, bodies): self.bodies = list(bodies) self.posts = [] + self.closed = False async def post( self, @@ -520,6 +521,10 @@ class ScriptedClient: self.posts.append(json) return httpx.Response(200, json = self.bodies[min(len(self.posts) - 1, len(self.bodies) - 1)]) + async def aclose(self): + # The Anthropic pass-through owns its client and closes it in a finally. + self.closed = True + async def _drive_non_streaming(monkeypatch, payload, bodies): import routes.inference as inf_mod @@ -867,7 +872,7 @@ class TestNudgeRetryAnthropic: from routes.inference import _anthropic_passthrough_non_streaming client = ScriptedClient(bodies) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) response = await _anthropic_passthrough_non_streaming( _llama_backend(), [{"role": "user", "content": "hi"}], @@ -925,7 +930,7 @@ class TestAnthropicPassthroughHealingText: from routes.inference import _anthropic_passthrough_non_streaming client = ScriptedClient([upstream]) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) response = await _anthropic_passthrough_non_streaming( _llama_backend(), [{"role": "user", "content": "hi"}], @@ -1171,7 +1176,7 @@ class TestAnthropicNonStreamingRoute: from routes.inference import _anthropic_passthrough_non_streaming client = ScriptedClient(bodies) - monkeypatch.setattr(inf_mod, "nonstreaming_client", lambda: client) + monkeypatch.setattr(inf_mod, "_cancelable_nonstreaming_client", lambda: client) response = await _anthropic_passthrough_non_streaming( _llama_backend(), [{"role": "user", "content": "hi"}], diff --git a/studio/backend/tests/test_safetensors_tool_loop.py b/studio/backend/tests/test_safetensors_tool_loop.py index bb18acf6e5..1043005f64 100644 --- a/studio/backend/tests/test_safetensors_tool_loop.py +++ b/studio/backend/tests/test_safetensors_tool_loop.py @@ -5051,3 +5051,27 @@ class TestFalseAlarmMarkerProse: assert [c[0] for c in exec_fn.calls] == ["web_search", "python"] assistant = next(m for m in convs[1] if m["role"] == "assistant") assert '"python"' not in (assistant.get("content") or "") + + +def test_both_tool_loops_say_they_are_waiting_for_approval(): + """A gated call must not report "Running" in either loop. + + The GGUF loop was fixed first and the safetensors one was missed, so the + badge counted up "Running ..." against a prompt nobody had answered yet. + Asserted on the source so the two paths cannot drift apart again. + """ + import ast + import os + + backend = os.path.join(os.path.dirname(__file__), "..") + for name in ("core/inference/safetensors_agentic.py", "core/inference/llama_cpp.py"): + with open(os.path.join(backend, name), encoding = "utf-8") as f: + tree = ast.parse(f.read()) + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "awaiting_approval_status" + ] + assert calls, f"{name} still announces a gated tool call as running" diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py index f91eec9817..3cc7d0604f 100644 --- a/studio/backend/tests/test_sf_client_tools_passthrough.py +++ b/studio/backend/tests/test_sf_client_tools_passthrough.py @@ -95,7 +95,7 @@ class _ScriptedBackend: for snap in snapshots: yield snap - def reset_generation_state(self): + def reset_generation_state(self, caller_cancel_event = None): self.reset_count += 1 diff --git a/studio/backend/tests/test_shutdown_preserves_live_worker.py b/studio/backend/tests/test_shutdown_preserves_live_worker.py index faf273411c..15ef93c002 100644 --- a/studio/backend/tests/test_shutdown_preserves_live_worker.py +++ b/studio/backend/tests/test_shutdown_preserves_live_worker.py @@ -9,6 +9,8 @@ holds sidecar transformers modules (breaking the rename on Windows). The methods the handle and return False so callers can refuse the swap. """ +import threading + import pytest from core.export.orchestrator import ExportOrchestrator @@ -52,6 +54,14 @@ def _bare_inference(): o._resp_queue = _Q() o._cancel_event = None o._drain_event = None + # Worker-scoped bookkeeping the teardown clears (see _reset_worker_scoped_state). + o._active_cancel_lock = threading.Lock() + o._active_cancel_events = [] + o._executing_cancel_events = [] + o._mailbox_lock = threading.Lock() + o._mailboxes = {} + o._direct_mailboxes = {} + o._request_cancel_events = {} return o diff --git a/studio/backend/tests/test_tool_sandbox_per_thread.py b/studio/backend/tests/test_tool_sandbox_per_thread.py new file mode 100644 index 0000000000..13bd95c9ed --- /dev/null +++ b/studio/backend/tests/test_tool_sandbox_per_thread.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Every conversation runs its tools in its own sandbox directory. + +Parallel chats lean on this: two conversations can be mid tool call at the same +time, so a shared working directory would let one overwrite the other's files. +The session id is the chat's thread id (or project- for project chats), and +the dir is derived from it here. + +HOME is redirected at import time, so nothing touches the real ~/studio_sandbox. +""" + +import os +import sys + +import pytest + +_backend = os.path.join(os.path.dirname(__file__), "..") +sys.path.insert(0, _backend) + + +@pytest.fixture +def workdir(tmp_path, monkeypatch): + """_get_workdir with HOME pointed at tmp_path and its cache cleared.""" + from core.inference import tools + + monkeypatch.setattr(os.path, "expanduser", lambda path: str(tmp_path)) + monkeypatch.setattr(tools, "_workdirs", {}) + return tools._get_workdir + + +def test_two_conversations_get_two_directories(workdir, tmp_path): + a = workdir("thread-alpha") + b = workdir("thread-beta") + assert a != b + assert os.path.basename(a) == "thread-alpha" + assert os.path.basename(b) == "thread-beta" + assert os.path.isdir(a) and os.path.isdir(b) + assert os.path.dirname(a) == os.path.dirname(b) == str(tmp_path / "studio_sandbox") + + +def test_the_same_conversation_keeps_its_directory(workdir): + # A later turn, or a tool continuation, must land back in the same place. + assert workdir("thread-alpha") == workdir("thread-alpha") + + +def test_a_directory_is_private_to_its_conversation(workdir): + a = workdir("thread-alpha") + b = workdir("thread-beta") + with open(os.path.join(a, "secret.txt"), "w", encoding = "utf-8") as f: + f.write("alpha") + assert os.listdir(b) == [] + + +def test_project_chats_deliberately_share_one_workspace(workdir, monkeypatch): + # Chats in a project are meant to see each other's files. + from core.inference import tools + monkeypatch.setattr(tools, "_get_project_workdir", lambda sid: "/tmp/project-ws") + assert tools._get_workdir("project-abc") == "/tmp/project-ws" + + +@pytest.mark.parametrize( + "session_id", + ["../escape", "a/b", "", " ", "x" * 65], +) +def test_a_session_id_cannot_escape_the_sandbox_root(workdir, tmp_path, session_id): + resolved = workdir(session_id) if session_id else workdir(None) + root = os.path.realpath(str(tmp_path / "studio_sandbox")) + assert os.path.realpath(resolved).startswith(root + os.sep) + assert os.path.basename(resolved) in {"_invalid", "_default"} + + +def test_no_session_id_falls_back_to_default(workdir): + assert os.path.basename(workdir(None)) == "_default" + + +@pytest.mark.skipif(sys.platform == "win32", reason = "POSIX permission bits") +def test_directories_are_private_to_the_user(workdir): + assert os.stat(workdir("thread-alpha")).st_mode & 0o777 == 0o700 diff --git a/studio/backend/tests/test_tool_xml_strip.py b/studio/backend/tests/test_tool_xml_strip.py index 941d9d044a..d02638f589 100644 --- a/studio/backend/tests/test_tool_xml_strip.py +++ b/studio/backend/tests/test_tool_xml_strip.py @@ -668,7 +668,8 @@ def test_route_history_and_passthrough_forward_the_display_gate(): blocks = { "safetensors history": r"Strip stale tool-call XML from prior assistant turns.*?\.strip\(\)", "anthropic history": r"Strip stale tool-call XML via the protected display helper.*?\.strip\(\)", - "anthropic passthrough": r"gated on the declared tools so an\n.*?\.strip\(\)", + # Anchored on the code, not the comment above it, so rewrapping prose cannot break this. + "anthropic passthrough": r"if not healing_active:.*?\.strip\(\)", } for label, pat in blocks.items(): m = _re.search(pat, _src, _re.DOTALL) diff --git a/studio/frontend/src/app/routes/__root.tsx b/studio/frontend/src/app/routes/__root.tsx index 7137fd6f96..9e72135bdd 100644 --- a/studio/frontend/src/app/routes/__root.tsx +++ b/studio/frontend/src/app/routes/__root.tsx @@ -12,6 +12,7 @@ import { import { ChatPage, clearNewChatDraft, + StopRunningChatsDialog, useChatRuntimeStore, type ChatSearch, } from "@/features/chat"; @@ -227,6 +228,8 @@ function RootLayout() { + {/* At the root, not under /chat: a swap can start from the Hub too. */} + {hideNavbar ? (
}> diff --git a/studio/frontend/src/components/app-sidebar.tsx b/studio/frontend/src/components/app-sidebar.tsx index a31d9b6ced..8aa4db99f4 100644 --- a/studio/frontend/src/components/app-sidebar.tsx +++ b/studio/frontend/src/components/app-sidebar.tsx @@ -520,15 +520,46 @@ export function AppSidebar() { }); const storeThreadId = useChatRuntimeStore((s) => s.activeThreadId); const setActiveThreadId = useChatRuntimeStore((s) => s.setActiveThreadId); - const anyChatRunning = useChatRuntimeStore((s) => - Object.values(s.runningByThreadId).some(Boolean), - ); - // The thread currently generating (if any), so "Return to Chat" lands on the - // live chat rather than an empty new-chat draft left active after New Chat. - const runningThreadId = useChatRuntimeStore((s) => { - const entry = Object.entries(s.runningByThreadId).find(([, on]) => on); - return entry ? entry[0] : null; - }); + // The whole map, so each row can show its own spinner. + const runningThreadIds = useChatRuntimeStore((s) => s.runningByThreadId); + // Rows, not raw thread ids: a compare conversation runs two pane threads but is one chat + // in the sidebar, so counting the map said "2 Chats" for a single compare row. + const runningChatCount = useMemo(() => { + const running = new Set( + Object.entries(runningThreadIds) + .filter(([, on]) => on) + .map(([id]) => id), + ); + if (running.size === 0) return 0; + let rows = 0; + for (const item of allChatItems) { + const ids = item.type === "compare" ? (item.threadIds ?? []) : [item.id]; + let claimed = false; + for (const id of ids) { + if (running.delete(id)) claimed = true; + } + if (claimed) rows += 1; + } + // Anything left belongs to no known row (a first turn mid-persist); count it as one. + return rows + running.size; + }, [runningThreadIds, allChatItems]); + const anyChatRunning = runningChatCount > 0; + // Where "Return to Chat" lands: the newest running chat, not the empty draft New Chat left + // active (map insertion order is start order). A compare row runs pane threads that /chat + // cannot address, so resolve those back to the pair id the route expects. + const runningTarget = useMemo(() => { + const ids = Object.entries(runningThreadIds) + .filter(([, on]) => on) + .map(([id]) => id); + const id = ids.length > 0 ? ids[ids.length - 1] : null; + if (!id) return null; + const pair = allChatItems.find( + (item) => item.type === "compare" && (item.threadIds ?? []).includes(id), + ); + return pair + ? { id: pair.id, compare: true as const } + : { id, compare: false as const }; + }, [runningThreadIds, allChatItems]); const activeThreadId = isChatRoute ? (search.thread as string | undefined) ?? (search.compare as string | undefined) ?? @@ -892,6 +923,12 @@ export function AppSidebar() { variant: "project" | "recent", ) { const isPinned = pinnedIdSet.has(item.id); + // A compare row's id is the pair id while runningByThreadId is keyed per pane thread, + // so aggregate its member threads instead. + const isGenerating = + item.type === "compare" + ? (item.threadIds ?? []).some((id) => Boolean(runningThreadIds[id])) + : Boolean(runningThreadIds[item.id]); const itemClass = variant === "project" ? "group/project-chat-item relative" @@ -951,6 +988,8 @@ export function AppSidebar() { data-testid="recent-thread" data-thread-type={item.type} data-thread-id={item.id} + data-generating={isGenerating ? "true" : undefined} + aria-busy={isGenerating || undefined} isActive={activeThreadId === item.id} className={buttonClass} onClick={() => { @@ -976,6 +1015,14 @@ export function AppSidebar() { {pendingRename?.id === item.id ? pendingRename.title : item.title} + {isGenerating && ( + + )} {variant === "project" && ( + ); +} + +function DownloadBtn({ code, name }: { code: string; name: string }) { + const download = useCallback(() => { + if (typeof document === "undefined") { + return; + } + try { + const blob = new Blob([code], { type: "text/plain;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = name; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + // Revoke next tick, after the click consumes the URL. + setTimeout(() => URL.revokeObjectURL(url), 0); + } catch { + // Never break the transcript over a download. + } + }, [code, name]); + + return ( + + ); +} + +/** A fence longer than any backtick run in the code, so a script containing ``` cannot end it early. */ +function fenceFor(source: string): string { + const longest = (source.match(/`+/g) ?? []).reduce( + (max, run) => Math.max(max, run.length), + 0, + ); + return "`".repeat(Math.max(3, longest + 1)); +} + +/** Syntax-highlighted code via Streamdown + shiki. Always in the DOM as plain monospace, but + * shiki only tokenizes once the block nears the viewport, so a long transcript does not + * highlight every script up front. Immediate where IntersectionObserver is missing. */ +function HighlightedCode({ + code: source, + language, + plain = false, +}: { + code: string; + language: string; + plain?: boolean; +}) { + const markdown = useMemo(() => { + const fence = fenceFor(source); + return `${fence}${language}\n${source}\n${fence}`; + }, [source, language]); + const containerRef = useRef(null); + const [nearViewport, setNearViewport] = useState( + () => typeof IntersectionObserver === "undefined", + ); + // Pinned to the bottom until the reader scrolls up, so a streaming payload visibly grows. + const pinnedToBottom = useRef(true); + useEffect(() => { + if (nearViewport) return; + const el = containerRef.current; + if (!el) return; + const io = new IntersectionObserver( + (entries) => { + if (entries.some((entry) => entry.isIntersecting)) { + setNearViewport(true); + io.disconnect(); + } + }, + // Highlight just before the block enters view, so it is ready on arrival. + { rootMargin: "200px" }, + ); + io.observe(el); + return () => io.disconnect(); + }, [nearViewport]); + + useEffect(() => { + const el = containerRef.current; + if (plain && el && pinnedToBottom.current) { + el.scrollTop = el.scrollHeight; + } + }, [plain, source]); + + const handleScroll = () => { + const el = containerRef.current; + if (el) { + pinnedToBottom.current = + el.scrollHeight - el.scrollTop - el.clientHeight < PIN_SLACK_PX; + } + }; + + // Skip shiki while the model is writing (it re-tokenizes every fragment) and on payloads too big. + const highlight = + nearViewport && !plain && source.length <= MAX_HIGHLIGHT_CHARS; + + return ( +
+ {highlight ? ( + + {markdown} + + ) : ( + // A div, not a 
: the container's [&_pre]:!p-0 would strip the padding and shift
+        // the content when shiki swaps in. whitespace-pre so long lines scroll.
+        

+          {source}
+        

+      )}
+
+ ); +} + +/** The code a tool is about to run, in the card's collapsible content so the chevron hides code and output together. */ +export function ToolCodeCell({ + label, + code, + language, + downloadName, + streaming = false, +}: { + label: string; + code: string; + language: string; + downloadName: string; + streaming?: boolean; +}) { + return ( +
+
+ + {label} + +
+ + +
+
+ +
+ ); +} diff --git a/studio/frontend/src/components/assistant-ui/tool-group.tsx b/studio/frontend/src/components/assistant-ui/tool-group.tsx index 469c449a70..af370d892e 100644 --- a/studio/frontend/src/components/assistant-ui/tool-group.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-group.tsx @@ -10,7 +10,11 @@ import { } from "react"; import { useAuiState } from "@assistant-ui/react"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; -import { toolOutputKey, useToolPaneScope } from "@/features/chat"; +import { + toolOutputKey, + useToolPaneScope, + useUnresolvedToolPaneScope, +} from "@/features/chat"; import { ChevronDownIcon } from "lucide-react"; import { Wrench01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; @@ -239,16 +243,23 @@ const ToolGroupImpl: FC< // Force the group open when any call is receiving tool_output events. const toolLiveOutput = useChatRuntimeStore((s) => s.toolLiveOutput); const paneScope = useToolPaneScope(); + const unresolvedScope = useUnresolvedToolPaneScope(); const hasLiveOutput = useAuiState(({ message }) => message.parts .slice(startIndex, endIndex + 1) .some( (part) => part.type === "tool-call" && - Object.prototype.hasOwnProperty.call( + // Either scope: a first turn writes under the unresolved one for its whole + // life, even after the autosave assigns the id (see useToolOutputFor). + (Object.prototype.hasOwnProperty.call( toolLiveOutput, toolOutputKey(paneScope, part.toolCallId), - ), + ) || + Object.prototype.hasOwnProperty.call( + toolLiveOutput, + toolOutputKey(unresolvedScope, part.toolCallId), + )), ), ); // Keep the group open once a confirmation or live output forced it (so an diff --git a/studio/frontend/src/components/assistant-ui/tool-live-output.tsx b/studio/frontend/src/components/assistant-ui/tool-live-output.tsx index 3434783f0a..df202b57f0 100644 --- a/studio/frontend/src/components/assistant-ui/tool-live-output.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-live-output.tsx @@ -4,7 +4,7 @@ "use client"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; -import { toolOutputKey, useToolPaneScope } from "@/features/chat"; +import { useToolOutputFor, useToolPaneScope } from "@/features/chat"; import { useEffect, useMemo, useRef } from "react"; import { tailText } from "./tool-result-output"; @@ -16,8 +16,10 @@ import { tailText } from "./tool-result-output"; */ export function ToolLiveOutput({ toolCallId }: { toolCallId: string }) { const paneScope = useToolPaneScope(); - const output = useChatRuntimeStore( - (s) => s.toolLiveOutput[toolOutputKey(paneScope, toolCallId)] ?? "", + const output = useToolOutputFor( + useChatRuntimeStore((s) => s.toolLiveOutput), + paneScope, + toolCallId, ); const scrollRef = useRef(null); // Pinned to the bottom until the user scrolls up (handler below), so diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx index 34e51b9d5a..e058a04ed1 100644 --- a/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-ui-python.tsx @@ -3,28 +3,25 @@ "use client"; -import { copyToClipboard } from "@/lib/copy-to-clipboard"; import { getAuthToken } from "@/features/auth/session"; import type { ToolCallMessagePartComponent } from "@assistant-ui/react"; import { useToolArgsStatus } from "@assistant-ui/react"; -import { code as codePlugin } from "@streamdown/code"; -import { CodeIcon, CopyIcon, DownloadIcon } from "lucide-react"; -import { Tick02Icon } from "@/lib/tick-icon"; -import { HugeiconsIcon } from "@hugeicons/react"; +import { CodeIcon } from "lucide-react"; import { Spinner } from "@/components/ui/spinner"; -import { memo, useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { Streamdown } from "streamdown"; +import { memo } from "react"; import { ToolFallbackContent, ToolFallbackRoot, ToolFallbackTrigger, } from "./tool-fallback"; +import { CopyBtn, ToolCodeCell } from "./tool-code-cell"; import { ToolLiveOutput } from "./tool-live-output"; import { ToolResultOutput } from "./tool-result-output"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; import { preferFullToolOutput, - toolOutputKey, + useToolAwaitingApproval, + useToolOutputFor, useToolPaneScope, } from "@/features/chat"; @@ -34,151 +31,6 @@ interface StructuredResult { sessionId: string; } -const MAX_DISPLAY = 10_000; -const COPY_RESET_MS = 2000; -const SHIKI_THEME = ["github-light", "github-dark"] as ["github-light", "github-dark"]; - -function truncate(text: string): string { - return text.length <= MAX_DISPLAY - ? text - : `${text.slice(0, MAX_DISPLAY)}\n... (truncated)`; -} - -function CopyBtn({ text }: { text: string }) { - const [copied, setCopied] = useState(false); - const timer = useRef | null>(null); - - useEffect(() => { - return () => { - if (timer.current) { - clearTimeout(timer.current); - } - }; - }, []); - - const copy = useCallback(async () => { - if (await copyToClipboard(text)) { - setCopied(true); - if (timer.current) { - clearTimeout(timer.current); - } - timer.current = setTimeout(() => setCopied(false), COPY_RESET_MS); - } - }, [text]); - - return ( - - ); -} - -/** Save the script as a .py file via a client-side Blob. */ -function DownloadBtn({ code, name = "script.py" }: { code: string; name?: string }) { - const download = useCallback(() => { - if (typeof document === "undefined") { - return; - } - try { - const blob = new Blob([code], { type: "text/x-python" }); - const url = URL.createObjectURL(blob); - const anchor = document.createElement("a"); - anchor.href = url; - anchor.download = name; - document.body.appendChild(anchor); - anchor.click(); - anchor.remove(); - // Revoke next tick, after the click consumes the URL. - setTimeout(() => URL.revokeObjectURL(url), 0); - } catch { - // Best-effort: never break the transcript over a download. - } - }, [code, name]); - - return ( - - ); -} - -/** Syntax-highlighted code via Streamdown + shiki; inherits parent container. - * The script is always in the DOM (a plain monospace placeholder), but shiki - * only tokenizes once the block scrolls near the viewport, so a long transcript - * with many scripts doesn't highlight every one up front. Falls back to - * immediate highlight when IntersectionObserver is unavailable (SSR / tests). */ -function HighlightedCode({ code: source, language }: { code: string; language: string }) { - const display = useMemo(() => truncate(source), [source]); - const markdown = useMemo( - () => `\`\`\`${language}\n${display}\n\`\`\``, - [display, language], - ); - const containerRef = useRef(null); - const [highlight, setHighlight] = useState( - () => typeof IntersectionObserver === "undefined", - ); - useEffect(() => { - if (highlight) return; - const el = containerRef.current; - if (!el) return; - const io = new IntersectionObserver( - (entries) => { - if (entries.some((entry) => entry.isIntersecting)) { - setHighlight(true); - io.disconnect(); - } - }, - // Highlight just before the block enters view so it's colorized by the - // time the user reaches it, without tokenizing off-screen scripts. - { rootMargin: "200px" }, - ); - io.observe(el); - return () => io.disconnect(); - }, [highlight]); - return ( -
- {highlight ? ( - - {markdown} - - ) : ( - // A div, not a 
: the container's [&_pre]:!p-0 would override a
-        // 
's padding and shift the content by p-3 when shiki swaps in. Keep
-        // the same p-3, and whitespace-pre (not pre-wrap) so long lines scroll in
-        // the container's overflow-auto exactly like the highlighted 
, rather
-        // than wrapping taller and then collapsing when shiki swaps in.
-        

-          {display}
-        

-      )}
-
- ); -} - function isStructuredResult(val: unknown): val is StructuredResult { return ( typeof val === "object" && @@ -221,46 +73,50 @@ const PythonToolUIImpl: ToolCallMessagePartComponent = ({ // Show the fuller live stream over a truncated result, keeping its exit // status. Session-transient: after a reload only the result remains. const paneScope = useToolPaneScope(); - const fullOutput = useChatRuntimeStore( - (s) => s.toolFullOutput[toolOutputKey(paneScope, toolCallId)] ?? "", + const fullOutput = useToolOutputFor( + useChatRuntimeStore((s) => s.toolFullOutput), + paneScope, + toolCallId, ); const displayOutput = preferFullToolOutput(fullOutput, output); const authToken = getAuthToken(); + // The gate only opens once the call parsed, so a pending approval means the script is + // written even while the args status still reads as streaming. + const awaitingApproval = useToolAwaitingApproval(toolCallId); + const isWriting = isWritingCode && !awaitingApproval; return ( - // Status/output collapse from history; the script source renders outside - // ToolFallbackContent so it stays visible on reopen (#7165). + // Script, status and output all collapse behind the one chevron. - {code && ( -
-
-
- - script - -
- - -
-
- -
-
- )} + {code && ( + + )}
{/* Output */} {isRunning ? ( <>
- {isWritingCode ? "Writing code…" : "Running…"} + + {awaitingApproval + ? "Waiting for approval…" + : isWriting + ? "Writing code…" + : "Running…"} +
{/* Live stdout streamed via tool_output SSE events. */} diff --git a/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx b/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx index 17ae26d388..b6ea2aaa6f 100644 --- a/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx +++ b/studio/frontend/src/components/assistant-ui/tool-ui-terminal.tsx @@ -3,69 +3,27 @@ "use client"; -import { copyToClipboard } from "@/lib/copy-to-clipboard"; import type { ToolCallMessagePartComponent } from "@assistant-ui/react"; import { useToolArgsStatus } from "@assistant-ui/react"; -import { CopyIcon, TerminalIcon } from "lucide-react"; -import { Tick02Icon } from "@/lib/tick-icon"; -import { HugeiconsIcon } from "@hugeicons/react"; +import { TerminalIcon } from "lucide-react"; import { Spinner } from "@/components/ui/spinner"; -import { memo, useCallback, useEffect, useRef, useState } from "react"; +import { memo } from "react"; import { ToolFallbackContent, ToolFallbackRoot, ToolFallbackTrigger, } from "./tool-fallback"; +import { CopyBtn, ToolCodeCell } from "./tool-code-cell"; import { ToolLiveOutput } from "./tool-live-output"; import { ToolResultOutput } from "./tool-result-output"; import { useChatRuntimeStore } from "@/features/chat/stores/chat-runtime-store"; import { preferFullToolOutput, - toolOutputKey, + useToolAwaitingApproval, + useToolOutputFor, useToolPaneScope, } from "@/features/chat"; -const COPY_RESET_MS = 2000; - -function CopyBtn({ text }: { text: string }) { - const [copied, setCopied] = useState(false); - const timer = useRef | null>(null); - - useEffect(() => { - return () => { - if (timer.current) { - clearTimeout(timer.current); - } - }; - }, []); - - const copy = useCallback(async () => { - if (await copyToClipboard(text)) { - setCopied(true); - if (timer.current) { - clearTimeout(timer.current); - } - timer.current = setTimeout(() => setCopied(false), COPY_RESET_MS); - } - }, [text]); - - return ( - - ); -} - const TerminalToolUIImpl: ToolCallMessagePartComponent = ({ toolCallId, args, @@ -87,13 +45,19 @@ const TerminalToolUIImpl: ToolCallMessagePartComponent = ({ // Show the fuller live stream over a truncated result, keeping its exit // status. Session-transient: after a reload only the result remains. const paneScope = useToolPaneScope(); - const fullOutput = useChatRuntimeStore( - (s) => s.toolFullOutput[toolOutputKey(paneScope, toolCallId)] ?? "", + const fullOutput = useToolOutputFor( + useChatRuntimeStore((s) => s.toolFullOutput), + paneScope, + toolCallId, ); const displayOutput = preferFullToolOutput(fullOutput, output); + // The gate only opens once the call parsed, so a pending approval means the command is + // written even while the args status still reads as streaming. + const awaitingApproval = useToolAwaitingApproval(toolCallId); + const isWriting = isWritingCommand && !awaitingApproval; return ( - // Open when mounted mid-run so live output shows; collapsed from history. + // Open mid-run so command and live output show, collapsed from history. + {command && ( + + )}
{isRunning ? ( <>
- {isWritingCommand ? "Writing command…" : "Running…"} + + {awaitingApproval + ? "Waiting for approval…" + : isWriting + ? "Writing command…" + : "Running…"} +
{/* Live stdout streamed via tool_output SSE events. */} diff --git a/studio/frontend/src/components/ui/spinner.tsx b/studio/frontend/src/components/ui/spinner.tsx index 283b4e21de..34543ed763 100644 --- a/studio/frontend/src/components/ui/spinner.tsx +++ b/studio/frontend/src/components/ui/spinner.tsx @@ -6,15 +6,22 @@ import { Loader2Icon } from "lucide-react"; import { cn } from "@/lib/utils"; -/** - * App-wide spinner: a clean circular arc with a rounded cap (lucide - * Loader2 / LoaderCircle), animated, inheriting the current text color. - */ -function Spinner({ className }: { className?: string }) { +/** App-wide spinner inheriting the current text color. `label` overrides the announcement + * where "loading" is not what it means (a sidebar chat is generating). */ +function Spinner({ + className, + label = "Loading", + "data-testid": dataTestId, +}: { + className?: string; + label?: string; + "data-testid"?: string; +}) { return ( ); diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts index a0be3ea640..b9e7229e34 100644 --- a/studio/frontend/src/features/chat/api/chat-adapter.ts +++ b/studio/frontend/src/features/chat/api/chat-adapter.ts @@ -59,6 +59,7 @@ import { shouldPreserveFullOutput, toolOutputKey, toolPaneScope, + toolThreadScope, } from "../tool-output-scope"; import type { ModelType } from "../types"; import { isMultimodalResponse } from "../types/api"; @@ -2232,7 +2233,20 @@ export function createOpenAIStreamAdapter( : undefined; const threadKey = resolvedThreadId; - runtime.setThreadRunning(threadKey, true); + // The run is durable on the server, but Stop, archive and delete reach a background + // thread only through this map: without a handle the supervisor kept planning against + // a deleted conversation. Registered before the run exists, since the thread can be + // stopped while createResearchRun is still in flight. + let researchRunId: string | null = null; + let researchStopRequested = false; + const researchServerCancel = () => { + researchStopRequested = true; + if (researchRunId) { + void cancelResearchRun(researchRunId).catch(() => {}); + } + }; + runtime.registerThreadServerCancel(threadKey, researchServerCancel); + runtime.setThreadRunning(threadKey, true, { owner: researchServerCancel }); let report = ""; let releaseResearchFollow: (() => void) | null = null; const researchFollowController = new AbortController(); @@ -2272,6 +2286,13 @@ export function createOpenAIStreamAdapter( blockedDomains: [...runtime.researchWebsitePolicy.blockedDomains], }, }); + researchRunId = createdRun.id; + if (researchStopRequested) { + // Stopped while createResearchRun was still in flight, so the handle had no + // id to act on. Replay it rather than following a run the user already ended. + void cancelResearchRun(createdRun.id).catch(() => {}); + return; + } releaseResearchFollow = beginExternalResearchFollow( createdRun, detachResearchFollow, @@ -2330,7 +2351,8 @@ export function createOpenAIStreamAdapter( } finally { abortSignal.removeEventListener("abort", forwardAdapterAbort); releaseResearchFollow?.(); - runtime.setThreadRunning(threadKey, false); + runtime.clearThreadServerCancel(threadKey, researchServerCancel); + runtime.setThreadRunning(threadKey, false, { owner: researchServerCancel }); } return; } @@ -2339,17 +2361,21 @@ export function createOpenAIStreamAdapter( ? `${sandboxSessionId || "_default"}:${resolvedThreadId}` : sandboxSessionId || "_default"; const toolConfirmationIdsByBackendId = new Map(); - // Store keys are pane-scoped since local tool ids ("call_0") repeat across - // turns and concurrent panes (compare mode). Track this run's keys so - // cleanup can't wipe another pane's. - const toolOutputPaneScope = toolPaneScope( - options.modelType, - options.pairId, + // Local tool ids ("call_0") repeat across turns, panes and conversations, so scope by pane + // AND thread. unstable_threadId alone, no activeThreadId fallback: the reader has only + // threadListItem.remoteId, which is exactly this value. + const toolOutputPaneScope = toolThreadScope( + toolPaneScope(options.modelType, options.pairId), + unstable_threadId, ); const scopedToolOutputKey = (id: string) => toolOutputKey(toolOutputPaneScope, id); const runToolLiveOutputKeys = new Set(); const resolvedThreadKey = resolvedThreadId ?? null; + // Which conversation was on screen when this run started. A first turn has no id yet, so + // this is the only way to tell later whether the user has switched away from it. + const activeThreadIdAtRunStart = + useChatRuntimeStore.getState().activeThreadId ?? null; const pendingImageEditReferenceForRun = runtime.pendingImageEditReference; const selectedImageEditReference = (pendingImageEditReferenceForRun?.threadId ?? null) === @@ -2755,8 +2781,11 @@ export function createOpenAIStreamAdapter( // waitForRunEnd resolves instead of hanging: this gate fires // before the streaming path's setThreadRunning(true). const gatedThreadKey = resolvedThreadId || "__default"; - runtime.setThreadRunning(gatedThreadKey, true); - runtime.setThreadRunning(gatedThreadKey, false); + // Own token: siblings share "__default", so an ownerless clear would drop their + // entries while they are still generating. + const gateOwner = () => {}; + runtime.setThreadRunning(gatedThreadKey, true, { owner: gateOwner }); + runtime.setThreadRunning(gatedThreadKey, false, { owner: gateOwner }); clearSelectedImageEditReference(); throw new Error(imageGateReason); } @@ -2774,13 +2803,44 @@ export function createOpenAIStreamAdapter( } const useAdapter = await resolveUseAdapter(resolvedThreadId, options); + const threadKey = resolvedThreadId || "__default"; + // A first turn files its handles under "__default"; autosave then assigns a real id and + // adoptDefaultThreadRun re-keys them mid-run. Resolve per use so later writes and the + // final clear follow the run instead of stranding entries behind. + const liveThreadKey = (owner: () => void) => + threadKey === "__default" + ? useChatRuntimeStore.getState().runKeyForOwner(threadKey, owner) + : threadKey; + + // Per-run token so a delayed stop POST can't match the next run. + const cancelId = + typeof crypto !== "undefined" && "randomUUID" in crypto + ? crypto.randomUUID() + : `${Date.now()}-${Math.random().toString(36).slice(2)}`; + + // Per-run abort, chained to assistant-ui's signal. cancelByThreadId only holds the visible + // thread's cancelRun(), so this controller is the only way to end a backgrounded chat's + // request; the cancel POST below reaches llama-server only. + const runAbort = new AbortController(); + const runSignal = runAbort.signal; + const forwardAbort = () => runAbort.abort(abortSignal.reason); + // Declared here, not at its registration below: it doubles as this run's identity token + // on the per-thread maps (see registerThreadServerCancel). + const serverCancel = () => runAbort.abort(); + if (abortSignal.aborted) { + forwardAbort(); + } else { + abortSignal.addEventListener("abort", forwardAbort, { once: true }); + } + // ── Audio model path (non-streaming) ───────────────────── const activeModel = runtime.models.find( (m) => m.id === params.checkpoint, ); if (activeModel?.isAudio && !activeModel?.hasAudioInput) { - const threadKey = resolvedThreadId || "__default"; - runtime.setThreadRunning(threadKey, true); + const audioCancel = () => runAbort.abort(); + runtime.registerThreadServerCancel(threadKey, audioCancel); + runtime.setThreadRunning(threadKey, true, { owner: audioCancel }); try { yield { content: [{ type: "text" as const, text: "Generating audio..." }], @@ -2790,6 +2850,10 @@ export function createOpenAIStreamAdapter( { model: params.checkpoint, messages: outboundMessages, + // Same run in both registries: without it the backend files this under no + // thread, and the stop-chats prompt counts the named local run and the + // unnamed backend one as two. + ...(resolvedThreadId ? { thread_id: resolvedThreadId } : {}), stream: false, temperature: params.temperature, top_p: params.topP, @@ -2800,7 +2864,7 @@ export function createOpenAIStreamAdapter( presence_penalty: params.presencePenalty, ...(useAdapter === undefined ? {} : { use_adapter: useAdapter }), }, - abortSignal, + runSignal, ); const audioUrl = `data:audio/wav;base64,${result.audio.data}`; @@ -2813,19 +2877,21 @@ export function createOpenAIStreamAdapter( ], }; } catch (err) { - if (!abortSignal.aborted) { + if (!runSignal.aborted) { toast.error("Audio generation failed", { description: err instanceof Error ? err.message : "Unknown error", }); } throw err; } finally { - runtime.setThreadRunning(threadKey, false); + abortSignal.removeEventListener("abort", forwardAbort); + const audioKey = liveThreadKey(audioCancel); + runtime.setThreadRunning(audioKey, false, { owner: audioCancel }); + runtime.clearThreadServerCancel(audioKey, audioCancel); } return; } - const threadKey = resolvedThreadId || "__default"; let waitingFirstChunk = true; let firstTokenSettled = false; const streamStartTime = Date.now(); @@ -2856,10 +2922,15 @@ export function createOpenAIStreamAdapter( const warmupDelayMs = 450; const warmupTimer = setTimeout(() => { if (!waitingFirstChunk) return; - if (abortSignal.aborted) return; + if (runSignal.aborted) return; runtime.setGeneratingStatus("waiting"); }, warmupDelayMs); - runtime.setThreadRunning(threadKey, true); + // Flagged local/external so the model-swap gate only counts the chats a reload ends; the + // backend leaves external-provider runs out of active_generations for the same reason. + runtime.setThreadRunning(threadKey, true, { + local: !isExternalRequest, + owner: serverCancel, + }); let cumulativeText = ""; let reasoningStartAt: number | null = null; let reasoningDuration = 0; @@ -3025,21 +3096,12 @@ export function createOpenAIStreamAdapter( timings?: ServerTimings; } | null = null; - // Per-run cancellation token so a delayed stop POST can't match - // the next run on the same thread. - const cancelId = - typeof crypto !== "undefined" && "randomUUID" in crypto - ? crypto.randomUUID() - : `${Date.now()}-${Math.random().toString(36).slice(2)}`; - // Colab-style proxies can swallow fetch aborts, so also POST // /inference/cancel explicitly on abort. const onAbortCancel = () => { - // assistant-ui aborts with AbortError(detach=true) when a thread's runtime - // unmounts (navigation / background thread switch) and detach=false for an - // explicit Stop. Only a real Stop cancels the backend run; a detach must - // leave a backgrounded generation streaming. - if ((abortSignal.reason as { detach?: boolean } | undefined)?.detach) { + // assistant-ui aborts with detach=true when a runtime unmounts and detach=false for an + // explicit Stop. Only a real Stop cancels the backend run; runSignal forwards the reason. + if ((runSignal.reason as { detach?: boolean } | undefined)?.detach) { return; } const body: Record = { cancel_id: cancelId }; @@ -3060,11 +3122,17 @@ export function createOpenAIStreamAdapter( keepalive: true, }).catch(() => {}); }; + + // Stop handle for when this conversation is not the visible one, which cancelByThreadId + // cannot reach. Aborting this run's own controller closes just its request, and the + // listener above posts its cancel_id so llama-server stops decoding too. For an + // external provider the abort is the stop, since its cancel_id is never registered. + runtime.registerThreadServerCancel(threadKey, serverCancel); try { - if (abortSignal.aborted) { + if (runSignal.aborted) { onAbortCancel(); } else { - abortSignal.addEventListener("abort", onAbortCancel, { once: true }); + runSignal.addEventListener("abort", onAbortCancel, { once: true }); } const { @@ -3536,7 +3604,7 @@ export function createOpenAIStreamAdapter( } clearSelectedImageEditReference(); await ThreadAutosaveHandle.awaitFirstSave(resolvedThreadId); - const stream = streamChatCompletions(requestPayload, abortSignal); + const stream = streamChatCompletions(requestPayload, runSignal); for await (const chunk of stream) { const chunkModel = (chunk as { model?: unknown }).model; @@ -3549,7 +3617,11 @@ export function createOpenAIStreamAdapter( chunk as unknown as { _toolStatus?: string } )._toolStatus; if (toolStatusText !== undefined) { - runtime.setToolStatus(toolStatusText || null); + runtime.setToolStatus( + liveThreadKey(serverCancel), + toolStatusText || null, + serverCancel, + ); continue; } @@ -3578,7 +3650,9 @@ export function createOpenAIStreamAdapter( } )._diffusionFrame; if (diffusionFrame !== undefined) { - runtime.setActiveDiffusionCanvas({ + // Keyed by thread so a background run's frames stay out of the visible chat + // instead of overwriting the frame it is painting. + runtime.setActiveDiffusionCanvas(liveThreadKey(serverCancel), { block: diffusionFrame.block ?? 0, step: diffusionFrame.step ?? 0, total: diffusionFrame.total ?? 0, @@ -3719,8 +3793,16 @@ export function createOpenAIStreamAdapter( const approvalId = (toolEvent.approval_id as string) || ""; const awaitingConfirmation = toolEvent.awaiting_confirmation === true; + // Reuse a provisional card's part id, else the confirmation-scoped id + // opens a second card and the first spins "Running" forever. + const openPartId = backendToolCallId + ? toolPartIdByBackendId.get(backendToolCallId) + : undefined; + const reuseOpenPart = + !!openPartId && + toolCallParts.some((p) => p.toolCallId === openPartId); const id = - awaitingConfirmation && approvalId + awaitingConfirmation && approvalId && !reuseOpenPart ? `${toolConfirmationScopeId}:${approvalId}` : backendToolCallId ? resolveToolPartId(backendToolCallId) @@ -4299,9 +4381,17 @@ export function createOpenAIStreamAdapter( // Anthropic-only (billed at the write premium). const cacheWriteTokens = meta?.usage?.cache_creation_input_tokens ?? 0; - // Gate on the captured checkpoint still being active so a late - // completion from provider A doesn't populate the bar after a - // mid-stream switch to provider B. + // Gate on the captured checkpoint so a late completion from provider A cannot populate + // the bar after a mid-stream switch to B, and on the captured thread so a background + // run finishing after New Chat cannot repaint another chat's usage. An unresolved run + // has no id to compare, so compare what was on screen when it started. A first turn is + // adopted onto an id mid-run and autosave moves activeThreadId with it, so read the + // adopted key, or the run stays "unresolved" for life and the bar stays blank. + const usageKey = liveThreadKey(serverCancel); + const usageThreadKey = usageKey === "__default" ? null : usageKey; + const usageThreadIsVisible = + useChatRuntimeStore.getState().activeThreadId === + (usageThreadKey ?? activeThreadIdAtRunStart); if ( meta?.usage && typeof meta.usage.prompt_tokens === "number" && @@ -4309,13 +4399,23 @@ export function createOpenAIStreamAdapter( typeof meta.usage.total_tokens === "number" && useChatRuntimeStore.getState().params.checkpoint === params.checkpoint ) { - useChatRuntimeStore.getState().setContextUsage({ + const usage = { promptTokens: meta.usage.prompt_tokens, completionTokens: meta.usage.completion_tokens, totalTokens: meta.usage.total_tokens, cachedTokens, cacheWriteTokens, - }); + }; + // File it under this run's own thread even when the gate below blocks the visible + // write, so switching back re-applies it. + if (usageThreadKey !== null) { + useChatRuntimeStore + .getState() + .setThreadContextUsage(usageThreadKey, usage); + } + if (usageThreadIsVisible) { + useChatRuntimeStore.getState().setContextUsage(usage); + } } const finishedAt = Date.now(); @@ -4368,7 +4468,7 @@ export function createOpenAIStreamAdapter( settleFirstTokenErr( err instanceof Error ? err : new Error("Generation failed"), ); - if (!abortSignal.aborted) { + if (!runSignal.aborted) { const msg = err instanceof Error ? err.message : String(err); if (err instanceof GenerationLengthError) { toast.error("Response ran out of tokens", { @@ -4406,13 +4506,18 @@ export function createOpenAIStreamAdapter( } throw err; } finally { - abortSignal.removeEventListener("abort", onAbortCancel); + runSignal.removeEventListener("abort", onAbortCancel); + abortSignal.removeEventListener("abort", forwardAbort); + // Resolve once: the clears below drop the owner the lookup keys on. + const cleanupKey = liveThreadKey(serverCancel); const confirmStore = useChatRuntimeStore.getState(); for (const part of toolCallParts) { confirmStore.clearToolConfirmation(part.toolCallId); } runtime.setGeneratingStatus(null); - runtime.setToolStatus(null); + // Scoped by thread AND by run: a global clear wiped every other running chat's badge, + // and an unowned one wiped a concurrent run's badge behind the same key. + runtime.setToolStatus(cleanupKey, null, serverCancel); // Clear only this run's live keys (a concurrent pane owns its own). A // key still here streamed stdout but never reached tool_end (SSE drop or // cancel), so promote it to full output first, else the partial @@ -4426,20 +4531,23 @@ export function createOpenAIStreamAdapter( store.clearToolLiveOutput(liveKey); } runToolLiveOutputKeys.clear(); - // Drop the transient denoising canvas so the finished bubble shows only - // the committed markdown answer (cancellation/error included). - runtime.setActiveDiffusionCanvas(null); + // Drop the transient denoising canvas so the finished bubble shows only the committed + // answer. Scoped: a global clear wiped another denoising chat's frame. + runtime.clearActiveDiffusionCanvasForThread(cleanupKey); clearTimeout(warmupTimer); if (waitingFirstChunk) { if (firstTokenSettled) { settleFirstTokenOk(); - } else if (abortSignal.aborted) { + } else if (runSignal.aborted) { settleFirstTokenErr(new Error("Cancelled")); } else { settleFirstTokenErr(new Error("No tokens received")); } } - runtime.setThreadRunning(threadKey, false); + // serverCancel narrows both clears: runs with no resolved thread id share the "__default" + // key, so a blind clear could drop a sibling's entry. + runtime.setThreadRunning(cleanupKey, false, { owner: serverCancel }); + runtime.clearThreadServerCancel(cleanupKey, serverCancel); } }, }; diff --git a/studio/frontend/src/features/chat/api/chat-api.ts b/studio/frontend/src/features/chat/api/chat-api.ts index 4f558545ca..a40867beea 100644 --- a/studio/frontend/src/features/chat/api/chat-api.ts +++ b/studio/frontend/src/features/chat/api/chat-api.ts @@ -129,6 +129,27 @@ export async function getApiMonitorEntry(id: string): Promise { return parseJsonOrThrow(response); } +export interface ActiveGenerationsResponse { + count: number; + /** Conversations with a generation in flight. Shorter than `count` when a + * first turn started before its thread id was persisted. */ + thread_ids: string[]; + /** One entry per in-flight request. `kind` is "chat" unless it is an + * embeddings / completions / audio call, which has no conversation. */ + active?: { thread_id: string | null; kind?: string }[]; + parallel_slots: number; +} + +/** + * Chats generating on the backend right now. Authoritative where `runningByThreadId` is not: + * that map is per-tab, empty after a reload and blind to a second tab, and /load and /unload + * 409 on these. + */ +export async function getActiveGenerations(): Promise { + const response = await authFetch("/api/inference/active-generations"); + return parseJsonOrThrow(response); +} + export async function loadModel( payload: LoadModelRequest, ): Promise { diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 7cc03fab26..8ae6c6fb15 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -2682,9 +2682,10 @@ export function ChatPage({ ggufNativeContextLength: null, activeNativePathToken: null, activeNativePathExpiresAtMs: null, - // Clear previous-model counters, else the relaxed external-provider - // render gate shows stale stats until the next completion. + // Clear previous-model counters, else the relaxed external-provider render gate shows + // stale stats. The per-thread copies go too, so a switch back cannot re-apply. contextUsage: null, + contextUsageByThreadId: {}, supportsReasoning: reasoningCaps.supportsReasoning, reasoningAlwaysOn: reasoningCaps.reasoningAlwaysOn, reasoningStyle: reasoningCaps.reasoningStyle, @@ -2906,7 +2907,13 @@ export function ChatPage({ ) { return; } - store.setContextUsage(usage); + // Key by the thread this restore read, like the history loader: the await above can + // outlast a switch away, and an unkeyed write would file this thread's usage under + // the incoming one. + store.setThreadContextUsage(threadId, usage); + if (store.activeThreadId === threadId) { + store.setContextUsage(usage); + } }) .catch((error) => { if (!isExpectedBackgroundChatStorageError(error)) { diff --git a/studio/frontend/src/features/chat/components/stop-running-chats-dialog.tsx b/studio/frontend/src/features/chat/components/stop-running-chats-dialog.tsx new file mode 100644 index 0000000000..dd9d6c13a1 --- /dev/null +++ b/studio/frontend/src/features/chat/components/stop-running-chats-dialog.tsx @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { useStopRunningChatsDialogStore } from "../stores/stop-running-chats-dialog-store"; + +/** + * Confirmation for applying a model or reload-required setting while chats are generating. + * They share one llama-server, so the swap ends all of them: name them and make the user + * opt in rather than truncating silently. + */ +export function StopRunningChatsDialog() { + const open = useStopRunningChatsDialogStore((s) => s.open); + const count = useStopRunningChatsDialogStore((s) => s.count); + const titles = useStopRunningChatsDialogStore((s) => s.titles); + const action = useStopRunningChatsDialogStore((s) => s.action); + const hasNonChat = useStopRunningChatsDialogStore((s) => s.hasNonChat); + const effect = useStopRunningChatsDialogStore((s) => s.effect); + const resolve = useStopRunningChatsDialogStore((s) => s.resolve); + + // Embeddings, raw completions and audio share the model but are not conversations, + // so name them generically rather than offering to stop chats that do not exist. + const noun = hasNonChat + ? count === 1 + ? "request" + : "requests" + : count === 1 + ? "chat" + : "chats"; + const sharer = hasNonChat ? "request" : "conversation"; + // Ejecting leaves no model loaded. Saying it "reloads the model" and offering "Stop and + // reload" promised the opposite of what confirming does, for the destructive one. + const unloads = effect === "unload"; + const lead = unloads + ? `${action || "Unloading the model"} leaves no model loaded, and every open ${sharer} shares it, ` + : `${action ? `${action} reloads the model, ` : "Reloading the model "}which every open ${sharer} shares, `; + const shown = titles.slice(0, 5); + const remaining = Math.max(0, titles.length - shown.length); + + return ( + { + // Escape / overlay click must resolve, or the caller's await hangs. + if (!next) resolve(false); + }} + > + + + + Stop {count} running {noun}? + + + {lead}so {count === 1 ? "this" : "these"} {noun} will stop + {hasNonChat ? "" : " generating"}. Work produced so far is kept. + + + {shown.length > 0 && ( +
    + {shown.map((title) => ( +
  • + {title} +
    • + ))} + {remaining > 0 && ( +
  • + and {remaining} more +
    • + )} +
+ )} + + resolve(false)}> + Keep generating + + resolve(true)}> + {unloads ? "Stop and unload" : "Stop and reload"} + + + + + ); +} diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts index 48a6168555..d4057591b0 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts @@ -28,6 +28,7 @@ import { validateModel, } from "../api/chat-api"; import { formatEta, formatRate } from "../utils/format-transfer"; +import { confirmStopRunningChatsIfNeeded } from "../utils/confirm-stop-running-chats"; import { GPU_LAYERS_AUTO, isLocalModelPath, @@ -463,7 +464,14 @@ export function useChatModelRuntime() { useChatRuntimeStore.getState().setModelLoading(true); void (async () => { try { + // Unforced on purpose: a chat may stream on the PREVIOUS model and must not be killed by + // cancelling this load. Nothing to report, since the route runs its stop-loading fast + // path ahead of the active-chat refusal. await unloadModel({ model_path: model.id }).catch(() => {}); + // clearCheckpoint above assumed nothing was left loaded, but a forced switch keeps the + // previous model resident until /load's teardown, and the stop-loading fast path leaves + // it there. Take the answer from the backend, which reports none once it was evicted. + await syncInferenceStatusToStore().catch(() => {}); } finally { cancelUnloadPendingRef.current = false; if (!loadingModelRef.current) { @@ -505,10 +513,11 @@ export function useChatModelRuntime() { // as a duplicate), don't start a second concurrent load and don't swallow the // request: surface it so the user waits or cancels. Centralized here so every // entry point is covered, not just the staged Load button. - const inFlightLoad = - loadingModelRef.current ?? - useChatRuntimeStore.getState().loadingModelPick; - if (inFlightLoad) { + const bailIfLoadInFlight = (): boolean => { + const inFlightLoad = + loadingModelRef.current ?? + useChatRuntimeStore.getState().loadingModelPick; + if (!inFlightLoad) return false; if (typeof selection !== "string" && selection.previousConfig) { applyPerModelConfigToRuntime(selection.previousConfig); } @@ -516,7 +525,7 @@ export function useChatModelRuntime() { inFlightLoad.id === modelId && (inFlightLoad.ggufVariant ?? null) === (ggufVariant ?? null) && (inFlightLoad.nativePathToken ?? null) === (nativePathToken ?? null); - if (loadingSamePick) return; + if (loadingSamePick) return true; const message = "Another model is already loading. Wait for it to finish or cancel it first."; setModelsError(message); @@ -524,8 +533,61 @@ export function useChatModelRuntime() { toast.info("Another model is already loading", { description: "Wait for it to finish or cancel it first.", }); + return true; + }; + if (bailIfLoadInFlight()) return; + + // Picking an external provider leaves the local model resident and stops the status poll + // mirroring it, so params.checkpoint cannot tell whether this pick is that same model. + // Ask the backend before prompting: /load answers already_loaded ahead of its cancel + // hook, so the dialog would promise to stop chats this pick never interrupts. A staged + // config always carries forceReload, so Apply still reloads and prompts. + const selectedCheckpoint = + useChatRuntimeStore.getState().params.checkpoint; + if (!forceReload && isExternalModelId(selectedCheckpoint)) { + const residentStatus = await getInferenceStatus().catch(() => null); + if ( + residentStatus && + resolveInferenceCheckpointId(residentStatus) === modelId && + (residentStatus.gguf_variant ?? null) === (ggufVariant ?? null) + ) { + // Same window as the confirm below: a rival load may have started during that GET, + // and it owns the resident model now. + if (bailIfLoadInFlight()) return; + // Roll back the config pre-applied for the load that is not happening BEFORE hydrating, + // so the resident model's status wins over the staged snapshot. + if (typeof selection !== "string" && selection.previousConfig) { + applyPerModelConfigToRuntime(selection.previousConfig); + } + const previousGgufVariant = + useChatRuntimeStore.getState().activeGgufVariant; + useChatRuntimeStore + .getState() + .setCheckpoint(modelId, residentStatus.gguf_variant); + applyActiveModelStatusToStore(residentStatus, { + previousCheckpoint: selectedCheckpoint, + previousGgufVariant, + }); + syncModelCapabilities(modelId, residentStatus); + return; + } + } + + // Every chat decodes on the llama-server this load replaces, so ask first, then allow the + // cancel; the 409 gate stays armed for callers that never confirmed. + const stopDecision = await confirmStopRunningChatsIfNeeded( + forceReload ? "Applying these settings" : "Loading a different model", + ); + if (!stopDecision.proceed) { + if (typeof selection !== "string" && selection.previousConfig) { + applyPerModelConfigToRuntime(selection.previousConfig); + } return; } + // Re-check: the confirm above awaits a GET, so a pick in that window would start a rival + // load over the same refs. Nothing awaits before the reservation below. + if (bailIfLoadInFlight()) return; + const forceCancelActive = stopDecision.forceCancelActive; const explicitIsLora = typeof selection === "string" ? undefined : selection.isLora; @@ -765,6 +827,10 @@ export function useChatModelRuntime() { upgrade: validation.transformers_upgrade, // No installable release: custom-code models may fall back to the trust_remote_code gate below. trustRemoteCodeFallback: validation.requires_trust_remote_code, + // The install refuses while chats generate and takes no force flag of its own, so + // without this the "Stop and reload" the user just confirmed dies here: Retry hits + // the same 409, and this path leaves chats running. + forceCancelActive, }); // The install unloads the previous model before the swap (even when // the swap then fails), so any exit after this point must roll back. @@ -808,7 +874,14 @@ export function useChatModelRuntime() { : undefined; if (currentCheckpoint) { - await unloadModel({ model_path: currentCheckpoint }); + // With chats generating, skip this preliminary unload: it cancels them ahead of /load's + // preflight, so a rejected target truncates replies for a model that never loads + // (/load evicts past those checks itself). Idle, unload first and free VRAM early. + if (!forceCancelActive) { + await unloadModel({ model_path: currentCheckpoint }); + } + // Set either way: /load can still leave no model resident, and an unneeded rollback + // hits already_loaded before the gate. previousWasUnloaded = true; } if (abortCtrl.signal.aborted) throw new Error("Cancelled"); @@ -915,6 +988,7 @@ export function useChatModelRuntime() { n_cpu_moe: loadNCpuMoe, tensor_split: loadSplitRatio ?? undefined, gpu_ids: loadSelectedGpuIds ?? undefined, + force_cancel_active: forceCancelActive, }); // If cancelled while loading, don't update UI to show @@ -1144,6 +1218,8 @@ export function useChatModelRuntime() { n_cpu_moe: stateBeforeUnload.loadedNCpuMoe ?? 0, tensor_split: stateBeforeUnload.loadedSplitRatio ?? undefined, gpu_ids: stateBeforeUnload.loadedGpuIds ?? undefined, + // The failed swap already unloaded the server those runs used. + force_cancel_active: true, }); const rollbackSpeculativeType = normalizeSpeculativeType( rollbackResponse.speculative_type, @@ -1540,13 +1616,15 @@ export function useChatModelRuntime() { if (!params.checkpoint) { return false; } - const runtime = useChatRuntimeStore.getState(); - if (runtime.modelLoading || runtime.loadingModelPick) { + const bailIfLoading = (): boolean => { + const runtime = useChatRuntimeStore.getState(); + if (!runtime.modelLoading && !runtime.loadingModelPick) return false; toast.info("A model is loading", { description: "Wait for it to finish or cancel it first.", }); - return false; - } + return true; + }; + if (bailIfLoading()) return false; setModelsError(null); if (isExternalModelId(params.checkpoint)) { clearCheckpoint(); @@ -1554,8 +1632,21 @@ export function useChatModelRuntime() { return true; } try { + // Ejecting tears down llama-server, so every chat stops. Same prompt, but it + // leaves no model loaded, so it must not be worded as a reload. + const stopDecision = await confirmStopRunningChatsIfNeeded( + "Unloading the model", + "unload", + ); + if (!stopDecision.proceed) return false; + // Same window as selectModel: a load may have started during the confirm. + if (bailIfLoading()) return false; + async function performUnload(): Promise { - await unloadModel({ model_path: params.checkpoint }); + await unloadModel({ + model_path: params.checkpoint, + force_cancel_active: stopDecision.forceCancelActive, + }); clearCheckpoint(); await refresh(); } diff --git a/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts b/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts index a08bd5fa54..651833102a 100644 --- a/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts +++ b/studio/frontend/src/features/chat/hooks/use-chat-sidebar-items.ts @@ -17,6 +17,7 @@ import { updateStoredChatThread, } from "../utils/chat-history-storage"; import { clearComposerDraft } from "../utils/composer-draft"; +import { stopChatThread } from "../utils/stop-chat-thread"; import { markChatThreadsDeleted, removeChatThreadTombstones, @@ -25,6 +26,8 @@ import { export interface SidebarItem { type: "single" | "compare"; id: string; + /** The pane threads behind this row id; `runningByThreadId` is keyed per pane thread. */ + threadIds?: string[]; title: string; createdAt: number; updatedAt: number; @@ -56,11 +59,13 @@ export function groupThreads( const existing = pairItems.get(t.pairId); if (existing) { existing.updatedAt = Math.max(existing.updatedAt, lastActivityAt(t)); + existing.threadIds?.push(t.id); continue; } const item: SidebarItem = { type: "compare", id: t.pairId, + threadIds: [t.id], title: t.title, createdAt: t.createdAt, updatedAt: lastActivityAt(t), @@ -160,10 +165,9 @@ export function useChatSidebarItems(options?: { } function cancelIfRunning(threadId: string): void { - const { runningByThreadId, cancelByThreadId } = - useChatRuntimeStore.getState(); - if (!runningByThreadId[threadId]) return; - cancelByThreadId[threadId]?.(); + // Reaches a background thread, which cancelByThreadId cannot: a deleted chat must stop, + // or the run keeps writing to a conversation that is gone. + stopChatThread(threadId); } export async function renameChatItem( diff --git a/studio/frontend/src/features/chat/index.ts b/studio/frontend/src/features/chat/index.ts index 2c1bbcefad..a96a7509bd 100644 --- a/studio/frontend/src/features/chat/index.ts +++ b/studio/frontend/src/features/chat/index.ts @@ -55,8 +55,12 @@ export { export { preferFullToolOutput, toolOutputKey, + toolThreadScope, + useToolOutputFor, + useUnresolvedToolPaneScope, useToolPaneScope, } from "./tool-output-scope"; +export { useToolAwaitingApproval } from "./tool-approval"; export { PermissionModeDropdown } from "./permission-mode-select"; export { useChatSearchStore } from "./stores/chat-search-store"; export { usePinnedChatsStore } from "./stores/pinned-chats-store"; @@ -80,6 +84,7 @@ export { export { ApiProviderLogo } from "./api-provider-logo"; export { useExternalProvidersStore } from "./stores/external-providers-store"; export { ChatSearchDialog } from "./components/chat-search-dialog"; +export { StopRunningChatsDialog } from "./components/stop-running-chats-dialog"; export { setTrainingCompareHandoff } from "./lib/training-compare-handoff"; export type { ProjectRecord } from "./types"; export { clearAllChats, countAllChats } from "./utils/clear-all-chats"; diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx index 2fa128bf2f..c22c753545 100644 --- a/studio/frontend/src/features/chat/runtime-provider.tsx +++ b/studio/frontend/src/features/chat/runtime-provider.tsx @@ -85,7 +85,11 @@ import { requestPromptQueueStop } from "./utils/prompt-queue-boundary"; import { isAssistantLocalThreadId } from "./utils/thread-ids"; const pendingHistoryAppendByMessageId = new Map>(); -const pendingRunStartReadyByMessageId = new Map>(); +// Resolves to the thread id assigned when this message's chat was first persisted. +const pendingRunStartReadyByMessageId = new Map< + string, + Promise +>(); type TitleResponse = { choices?: Array<{ @@ -699,6 +703,10 @@ function createStudioDbAdapter( async initialize(threadId: string) { await ensureThreadRecord({ threadId, modelType, pairId, projectId }); + // A run already streaming on this thread filed its handles under "__default" because + // the id did not exist yet. Re-key them now, or the sidebar row and Stop look up an + // id nothing is registered against. + useChatRuntimeStore.getState().adoptDefaultThreadRun(threadId); return { remoteId: threadId, externalId: undefined }; }, @@ -835,8 +843,8 @@ function trackHistoryAppend( function trackRunStartReady( messageId: string, - ready: Promise, -): Promise { + ready: Promise, +): Promise { pendingRunStartReadyByMessageId.set(messageId, ready); const cleanup = () => { setTimeout(() => { @@ -851,7 +859,7 @@ function trackRunStartReady( async function waitForRunStartHistoryAppend( messages: Parameters[0]["messages"], -): Promise { +): Promise { // Deep Research reserves an assistant placeholder before invoking the model // adapter, so the user message is not necessarily the final entry here. const userMessage = [...messages] @@ -862,15 +870,16 @@ async function waitForRunStartHistoryAppend( } const runStartReady = pendingRunStartReadyByMessageId.get(userMessage.id); const historyAppendReady = pendingHistoryAppendByMessageId.get(userMessage.id); - const pending = [runStartReady, historyAppendReady].filter( - (ready): ready is Promise => ready !== undefined, - ); - if (pending.length === 0) { - return; + if (runStartReady === undefined && historyAppendReady === undefined) { + return undefined; } let didBecomeReady = false; + let adoptedThreadId: string | undefined; try { - await Promise.all(pending); + [adoptedThreadId] = await Promise.all([ + runStartReady ?? Promise.resolve(undefined), + historyAppendReady?.then(() => undefined), + ]); didBecomeReady = true; } finally { if ( @@ -881,14 +890,22 @@ async function waitForRunStartHistoryAppend( pendingRunStartReadyByMessageId.delete(userMessage.id); } } + return adoptedThreadId; } function createPersistedRunAdapter(adapter: ChatModelAdapter): ChatModelAdapter { return { ...adapter, async *run(options) { - await waitForRunStartHistoryAppend(options.messages); - const result = adapter.run(options); + const adoptedThreadId = await waitForRunStartHistoryAppend(options.messages); + // The thread has an id by the time that resolves, but assistant-ui bound unstable_threadId + // before the await. Hand the run its real id so a first turn never files its handles + // under the unresolved key that concurrent runs share. + const result = adapter.run( + !options.unstable_threadId && adoptedThreadId + ? { ...options, unstable_threadId: adoptedThreadId } + : options, + ); if (!result) { return; } @@ -1153,7 +1170,13 @@ function useStudioRuntimeAdapters( : typeof store.ggufContextLength === "number" && store.ggufContextLength > 0; if (savedUsage && withinLocalLimit && modelMatches) { - store.setContextUsage(savedUsage); + // Key by the thread this loader read, not whichever is active when the await resolves: + // a switch inside it would file this thread's usage under the incoming one. Same rule + // the adapter's end-of-run write follows. + store.setThreadContextUsage(remoteId, savedUsage); + if (store.activeThreadId === remoteId) { + store.setContextUsage(savedUsage); + } } // If any message has a stored parentId, reconstruct the tree so @@ -1179,7 +1202,10 @@ function useStudioRuntimeAdapters( append({ parentId, message }: ExportedMessageRepositoryItem) { const initializeThread = aui.threadListItem().initialize(); - trackRunStartReady(message.id, initializeThread.then(() => undefined)); + trackRunStartReady( + message.id, + initializeThread.then(({ remoteId }) => remoteId), + ); const write = (async () => { const { remoteId } = await initializeThread; if (isChatThreadDeleted(remoteId)) { @@ -1308,17 +1334,6 @@ function createRuntimeHook(modelType: ModelType, pairId?: string) { }; } -function stopChatRun(threadId: string | null | undefined) { - if (!threadId) { - return; - } - try { - useChatRuntimeStore.getState().cancelByThreadId[threadId]?.(); - } catch { - // The run may have ended while navigation was mounting. - } -} - function ThreadAutoSwitch({ threadId, syncActiveThreadId = true, @@ -1333,8 +1348,9 @@ function ThreadAutoSwitch({ useEffect(() => { if (!isLoading && mainThreadId !== threadId) { if (syncActiveThreadId) { - requestPromptQueueStop(); - stopChatRun(mainThreadId); + // Stop queueing prompts to the outgoing thread but leave its run alone: its runtime + // stays mounted and keeps streaming. Only an explicit Stop cancels one. + requestPromptQueueStop({ cancelActiveRun: false }); } const switchResult = aui.threads().switchToThread(threadId) as unknown; if ( @@ -1365,16 +1381,14 @@ function ThreadNewChatSwitch({ }: { nonce: string }): ReactElement | null { const aui = useAui(); const isLoading = useAuiState(({ threads }) => threads.isLoading); - const mainThreadId = useAuiState(({ threads }) => threads.mainThreadId); - const mainThreadIdRef = useRef(mainThreadId); - mainThreadIdRef.current = mainThreadId; - + // The outgoing thread is not read here: New Chat leaves it running. useEffect(() => { if (isLoading) { return; } - requestPromptQueueStop(); - stopChatRun(mainThreadIdRef.current); + // New Chat leaves the previous conversation generating: its runtime stays mounted and + // the sidebar spins. Stopping it is its own Stop button's job. + requestPromptQueueStop({ cancelActiveRun: false }); // Switch to a fresh local thread without persisting it yet; persistence // still happens on first message append. void aui.threads().switchToNewThread(); diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index 237cd857f0..98b8676c10 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -762,6 +762,30 @@ export function isDownloadableHubRepo(x: { ); } +type ContextUsageSnapshot = { + promptTokens: number; + completionTokens: number; + totalTokens: number; + cachedTokens: number; + // Anthropic-only; optional so pre-cache-stats persisted entries load. + cacheWriteTokens?: number; +}; + +/** + * One live run behind `runningByThreadId[id]`, with the `local` flag it started with so the + * model-swap gate can tell llama-server runs from external ones when runs share a key. + */ +type ThreadRunOwner = { + owner: () => void; + local: boolean; +}; + +type ToolStatusEntry = { + status: string; + startedAt: number; + owner?: () => void; +}; + type ChatRuntimeStore = { settingsHydrated: boolean; params: InferenceParams; @@ -771,7 +795,25 @@ type ChatRuntimeStore = { models: ChatModelSummary[]; loras: ChatLoraSummary[]; runningByThreadId: Record; + /** + * The subset of `runningByThreadId` decoding on the local llama-server. Swapping the local + * model neither interrupts an external-provider chat nor needs its consent, which is why + * the backend keeps those out of `active_generations` too. + */ + localRunByThreadId: Record; + /** + * Which runs set `runningByThreadId[id]`; see `setThreadRunning`'s `owner`. A list, not one + * entry: runs without a resolved thread id share the "__default" key, so one entry would let + * a newer run's clear delete an older run's flag while it still generates. + */ + runOwnerByThreadId: Record; cancelByThreadId: Record void>; + /** + * Backend cancels for the threads generating in the background. `cancelByThreadId` only holds + * the visible thread's `cancelRun()`, so the adapter parks a closure here that POSTs that + * run's own cancel_id. A list for the same reason as `runOwnerByThreadId`: "__default" is shared. + */ + serverCancelByThreadId: Record void)[]>; autoTitle: boolean; hfToken: string; modelsError: string | null; @@ -892,7 +934,16 @@ type ChatRuntimeStore = { * consulted when `providerSupportsBuiltinWebFetch` is true. */ webFetchToolsEnabled: boolean; - toolStatus: string | null; + /** + * Live tool status per conversation ("Running Python: ...") with its start time. Keyed by + * thread, or one chat's tool call shows above every other composer; the timestamp keeps the + * counter running across a thread switch. + */ + /** + * Per-run entries, newest last. Unresolved threads share "__default", so one scalar per key + * meant a finishing run's clear removed a sibling's status while its tool was still running. + */ + toolStatusByThreadId: Record; /** Live stdout/stderr from running tools, keyed by toolCallId. Transient: * appended by tool_output, cleared on tool_end or run end. */ toolLiveOutput: Record; @@ -959,9 +1010,12 @@ type ChatRuntimeStore = { /** Active model is a block-diffusion model (DiffusionGemma): drives the * denoising-canvas artifact auto-render. */ loadedIsDiffusion: boolean; - /** Live denoising frame for the in-progress diffusion message. Transient: set - * per step, cleared when the run ends, never persisted into the transcript. */ - activeDiffusionCanvas: DiffusionCanvasFrame | null; + /** + * Live denoising frame per conversation ("__default" until the id exists). Transient: set per + * step, cleared when the run ends, never persisted. Keyed, not global: two denoising chats + * overwrote each other's frame, so the visible preview flickered or vanished. + */ + activeDiffusionCanvasByThreadId: Record; customContextLength: number | null; /** The pinned context the loaded model used (null = Auto), so dirty-tracking * and a later fit Apply can tell an explicit pin apart from Auto. */ @@ -984,14 +1038,13 @@ type ChatRuntimeStore = { pendingAudioBase64: string | null; pendingAudioName: string | null; pendingImageEditReference: PendingImageEditReference | null; - contextUsage: { - promptTokens: number; - completionTokens: number; - totalTokens: number; - cachedTokens: number; - // Anthropic-only; optional so pre-cache-stats persisted entries load. - cacheWriteTokens?: number; - } | null; + contextUsage: ContextUsageSnapshot | null; + /** + * Per-thread copy of the above, so the bar survives a switch away and back. `contextUsage` is + * the VISIBLE conversation's usage and a background run may not write it, so without this a + * run finishing off-screen leaves nothing to restore. + */ + contextUsageByThreadId: Record; modelLoading: boolean; loadingModelPick: LoadingModelPick | null; activeNativePathToken: string | null; @@ -1010,9 +1063,35 @@ type ChatRuntimeStore = { setActivePresetSource: (source: ChatPresetSource) => void; setModels: (models: ChatModelSummary[]) => void; setLoras: (loras: ChatLoraSummary[]) => void; - setThreadRunning: (threadId: string, running: boolean) => void; + /** + * `local` defaults to true, so an unqualified caller still counts for the model-swap gate. + * `owner` narrows the clear to the run that set the flag: unresolved thread ids share the + * "__default" key, so a blind delete would drop a sibling's live entry. Owners accumulate, + * so the flag survives until the last one clears. + */ + setThreadRunning: ( + threadId: string, + running: boolean, + options?: { local?: boolean; owner?: () => void }, + ) => void; + /** + * Re-key a first turn's run handles once its thread is persisted. + * + * A run that starts before its id exists files everything under "__default". Nothing moved it + * afterwards, so once the user navigated away the sidebar found no run and showed no spinner; + * stopChatThread had no handle either and the generation carried on holding a slot. + */ + adoptDefaultThreadRun: (threadId: string) => void; + /** + * Which key this run's handles live under now. `adoptDefaultThreadRun` re-keys them mid-run, + * so a run that started under "__default" must look its owner up instead of reusing the key + * it captured, or its writes and its final clear miss the entries. + */ + runKeyForOwner: (fallbackKey: string, owner: () => void) => string; registerThreadCancel: (threadId: string, cancel: () => void) => void; clearThreadCancel: (threadId: string) => void; + registerThreadServerCancel: (threadId: string, cancel: () => void) => void; + clearThreadServerCancel: (threadId: string, cancel?: () => void) => void; setAutoTitle: (enabled: boolean) => void; setHfToken: (token: string) => void; setModelsError: (error: string | null) => void; @@ -1066,7 +1145,15 @@ type ChatRuntimeStore = { setRagAutoInjectMinScore: (score: number) => void; setRagOcrScanned: (enabled: boolean) => void; setRagCaptionFigures: (enabled: boolean) => void; - setToolStatus: (status: string | null) => void; + /** + * `owner` is the run's identity token, as for `setThreadRunning`: unresolved threads share + * "__default", so without it one run's cleanup clears a concurrent run's status. + */ + setToolStatus: ( + threadId: string, + status: string | null, + owner?: () => void, + ) => void; appendToolLiveOutput: (toolCallId: string, text: string) => void; /** Clear one tool's live output, or all when no id is given. */ clearToolLiveOutput: (toolCallId?: string) => void; @@ -1075,7 +1162,13 @@ type ChatRuntimeStore = { /** Drop a stale preserved full output (a new run is reusing the id). */ clearToolFullOutput: (toolCallId: string) => void; setGeneratingStatus: (status: string | null) => void; - setActiveDiffusionCanvas: (canvas: DiffusionCanvasFrame | null) => void; + setActiveDiffusionCanvas: ( + threadId: string | null, + canvas: DiffusionCanvasFrame, + ) => void; + /** Drop only `threadId`'s canvas: a run ending in a background chat must not wipe the + * frame another chat is still painting. */ + clearActiveDiffusionCanvasForThread: (threadId: string | null) => void; setAutoHealToolCalls: (enabled: boolean) => void; setNudgeToolCalls: (enabled: boolean) => void; setMaxToolCallsPerMessage: (value: number) => void; @@ -1095,6 +1188,11 @@ type ChatRuntimeStore = { ) => void; clearPendingImageEditReference: () => void; setContextUsage: (usage: ChatRuntimeStore["contextUsage"]) => void; + /** A finished run's usage, kept per thread so switching back re-applies it. */ + setThreadContextUsage: ( + threadId: string, + usage: ContextUsageSnapshot, + ) => void; }; type PersistedChatSettings = Awaited< @@ -1310,7 +1408,10 @@ export const useChatRuntimeStore = create((set, get) => ({ models: [], loras: [], runningByThreadId: {}, + localRunByThreadId: {}, + runOwnerByThreadId: {}, cancelByThreadId: {}, + serverCancelByThreadId: {}, autoTitle: false, hfToken: useHfTokenStore.getState().token, modelsError: null, @@ -1374,11 +1475,11 @@ export const useChatRuntimeStore = create((set, get) => ({ ), ragOcrScanned: loadBool(CHAT_RAG_OCR_KEY, DEFAULT_RAG_OCR), ragCaptionFigures: loadBool(CHAT_RAG_CAPTION_KEY, DEFAULT_RAG_CAPTION), - toolStatus: null, + toolStatusByThreadId: {}, toolLiveOutput: {}, toolFullOutput: {}, generatingStatus: null, - activeDiffusionCanvas: null, + activeDiffusionCanvasByThreadId: {}, autoHealToolCalls: true, nudgeToolCalls: true, maxToolCallsPerMessage: 25, @@ -1423,6 +1524,7 @@ export const useChatRuntimeStore = create((set, get) => ({ pendingAudioName: null, pendingImageEditReference: null, contextUsage: null, + contextUsageByThreadId: {}, modelLoading: false, loadingModelPick: null, activeNativePathToken: null, @@ -1495,7 +1597,9 @@ export const useChatRuntimeStore = create((set, get) => ({ const checkpointChanged = state.params.checkpoint !== params.checkpoint; return { params, - ...(checkpointChanged ? { contextUsage: null } : {}), + ...(checkpointChanged + ? { contextUsage: null, contextUsageByThreadId: {} } + : {}), }; }), setCustomPresets: (customPresets) => @@ -1518,16 +1622,94 @@ export const useChatRuntimeStore = create((set, get) => ({ }), setModels: (models) => set({ models }), setLoras: (loras) => set({ loras }), - setThreadRunning: (threadId, running) => + setThreadRunning: (threadId, running, options) => set((state) => { const next = { ...state.runningByThreadId }; + const nextLocal = { ...state.localRunByThreadId }; + const nextOwner = { ...state.runOwnerByThreadId }; + const owners = state.runOwnerByThreadId[threadId] ?? []; + const local = options?.local !== false; if (running) { next[threadId] = true; + if (options?.owner) { + nextOwner[threadId] = [...owners, { owner: options.owner, local }]; + } + // Any local owner keeps the key counted by the model-swap gate, so an external run + // joining a shared key must not clear a sibling's flag. + if (local) { + nextLocal[threadId] = true; + } else if (!owners.some((o) => o.local)) { + delete nextLocal[threadId]; + } } else { - delete next[threadId]; + const remaining = options?.owner + ? owners.filter((o) => o.owner !== options.owner) + : []; + // An owner missing from the list was already cleared, or the key belongs to siblings + // only: either way this run must change nothing. + if (options?.owner && remaining.length === owners.length) return state; + // An ownerless clear predates per-run tracking, so it must not speak for runs that + // own the key: leave them to clear themselves. + if (!options?.owner && owners.length > 0) return state; + if (remaining.length > 0) { + nextOwner[threadId] = remaining; + if (remaining.some((o) => o.local)) { + nextLocal[threadId] = true; + } else { + delete nextLocal[threadId]; + } + } else { + delete next[threadId]; + delete nextLocal[threadId]; + delete nextOwner[threadId]; + } } - return { runningByThreadId: next }; + return { + runningByThreadId: next, + localRunByThreadId: nextLocal, + runOwnerByThreadId: nextOwner, + }; }), + adoptDefaultThreadRun: (threadId) => + set((state) => { + const key = "__default"; + if (!threadId || threadId === key) return state; + // Two first turns can share "__default", and nothing links a run there to the thread being + // persisted. Moving the arrays wholesale handed this thread the sibling's owner and stop + // handle too, so stopping one aborted both. Adopt only when the key holds a single run. + if ((state.runOwnerByThreadId[key]?.length ?? 0) > 1) return state; + // Only the transient run maps move. Anything already filed under the real id wins, + // since that is a later, better-identified run. + const moved: Partial = {}; + const move = ( + map: Record, + name: keyof ChatRuntimeStore, + ) => { + const entry = map[key]; + if (entry === undefined || map[threadId] !== undefined) return; + const next = { ...map }; + delete next[key]; + next[threadId] = entry; + (moved as Record)[name as string] = next; + }; + move(state.runningByThreadId, "runningByThreadId"); + move(state.localRunByThreadId, "localRunByThreadId"); + move(state.runOwnerByThreadId, "runOwnerByThreadId"); + move(state.cancelByThreadId, "cancelByThreadId"); + move(state.serverCancelByThreadId, "serverCancelByThreadId"); + move(state.toolStatusByThreadId, "toolStatusByThreadId"); + move( + state.activeDiffusionCanvasByThreadId, + "activeDiffusionCanvasByThreadId", + ); + return Object.keys(moved).length > 0 ? moved : state; + }), + runKeyForOwner: (fallbackKey, owner) => { + for (const [key, entries] of Object.entries(get().runOwnerByThreadId)) { + if (entries.some((e) => e.owner === owner)) return key; + } + return fallbackKey; + }, registerThreadCancel: (threadId, cancel) => set((state) => { const next = { ...state.cancelByThreadId }; @@ -1541,6 +1723,29 @@ export const useChatRuntimeStore = create((set, get) => ({ delete next[threadId]; return { cancelByThreadId: next }; }), + registerThreadServerCancel: (threadId, cancel) => + set((state) => { + const next = { ...state.serverCancelByThreadId }; + next[threadId] = [...(state.serverCancelByThreadId[threadId] ?? []), cancel]; + return { serverCancelByThreadId: next }; + }), + // `cancel` narrows removal to the run that registered it: unresolved thread ids share the + // "__default" key, so a blind delete would drop a live sibling. + clearThreadServerCancel: (threadId, cancel) => + set((state) => { + const current = state.serverCancelByThreadId[threadId]; + if (current === undefined) return state; + const remaining = + cancel === undefined ? [] : current.filter((c) => c !== cancel); + if (remaining.length === current.length) return state; + const next = { ...state.serverCancelByThreadId }; + if (remaining.length > 0) { + next[threadId] = remaining; + } else { + delete next[threadId]; + } + return { serverCancelByThreadId: next }; + }), setAutoTitle: (autoTitle) => set((state) => { setScalarSettingVersion("autoTitle", autoTitle, state.autoTitle); @@ -1588,14 +1793,24 @@ export const useChatRuntimeStore = create((set, get) => ({ maxTokens: nextMaxTokens, }, activeGgufVariant: ggufVariant ?? null, - ...(checkpointChanged ? { contextUsage: null } : {}), + ...(checkpointChanged + ? { contextUsage: null, contextUsageByThreadId: {} } + : {}), // Switching to an external provider disables Deep Research, which only // applies to the local base model. ...(isExternalModelId(modelId) ? { deepResearchEnabled: false } : {}), }; }), + // Re-apply the incoming thread's own usage rather than blanking the bar: a run that finished + // in the background never wrote the visible value, and a still-mounted runtime skips the + // history loader on the way back. setActiveThreadId: (activeThreadId) => - set({ activeThreadId, contextUsage: null }), + set((state) => ({ + activeThreadId, + contextUsage: activeThreadId + ? (state.contextUsageByThreadId[activeThreadId] ?? null) + : null, + })), setActiveProjectId: (activeProjectId) => set({ activeProjectId }), setIncognito: (incognito) => { if (incognito) saveBool(CHAT_DEEP_RESEARCH_ENABLED_KEY, false); @@ -1626,6 +1841,7 @@ export const useChatRuntimeStore = create((set, get) => ({ ggufNativeContextLength: null, modelRequiresTrustRemoteCode: false, contextUsage: null, + contextUsageByThreadId: {}, supportsReasoning: false, reasoningAlwaysOn: false, reasoningEnabled: true, @@ -1647,10 +1863,10 @@ export const useChatRuntimeStore = create((set, get) => ({ webFetchToolsEnabled: false, // Only the per-session enable pill resets; source/mode/top_k persist. ragEnabled: false, - toolStatus: null, + toolStatusByThreadId: {}, toolLiveOutput: {}, toolFullOutput: {}, - activeDiffusionCanvas: null, + activeDiffusionCanvasByThreadId: {}, kvCacheDtype: null, loadedKvCacheDtype: null, speculativeType: readPersistedSpeculativeType(), @@ -1945,7 +2161,31 @@ export const useChatRuntimeStore = create((set, get) => ({ saveBool(CHAT_RAG_CAPTION_KEY, ragCaptionFigures); return { ragCaptionFigures }; }), - setToolStatus: (toolStatus) => set({ toolStatus }), + setToolStatus: (threadId, status, owner) => + set((state) => { + const next = { ...state.toolStatusByThreadId }; + const entries = state.toolStatusByThreadId[threadId] ?? []; + const mine = entries.find((e) => e.owner === owner); + if (!status) { + // Drop only this run's entry: a sibling behind the same key may still be running a tool, + // and its status has to survive this clear. + if (mine === undefined) return state; + const rest = entries.filter((e) => e !== mine); + if (rest.length > 0) { + next[threadId] = rest; + } else { + delete next[threadId]; + } + } else { + // Same text from the same run means the same call, so keep startedAt: only a new tool restarts it. + if (mine?.status === status) return state; + const entry = { status, startedAt: Date.now(), owner }; + next[threadId] = mine + ? entries.map((e) => (e === mine ? entry : e)) + : [...entries, entry]; + } + return { toolStatusByThreadId: next }; + }), appendToolLiveOutput: (toolCallId, text) => set((state) => ({ toolLiveOutput: { @@ -1983,8 +2223,21 @@ export const useChatRuntimeStore = create((set, get) => ({ delete next[toolCallId]; return { toolLiveOutput: next }; }), - setActiveDiffusionCanvas: (activeDiffusionCanvas) => - set({ activeDiffusionCanvas }), + setActiveDiffusionCanvas: (threadId, canvas) => + set((state) => ({ + activeDiffusionCanvasByThreadId: { + ...state.activeDiffusionCanvasByThreadId, + [threadId || "__default"]: canvas, + }, + })), + clearActiveDiffusionCanvasForThread: (threadId) => + set((state) => { + const key = threadId || "__default"; + if (state.activeDiffusionCanvasByThreadId[key] === undefined) return state; + const next = { ...state.activeDiffusionCanvasByThreadId }; + delete next[key]; + return { activeDiffusionCanvasByThreadId: next }; + }), setGeneratingStatus: (generatingStatus) => set({ generatingStatus }), setAutoHealToolCalls: (autoHealToolCalls) => set((state) => { @@ -2050,7 +2303,27 @@ export const useChatRuntimeStore = create((set, get) => ({ set({ pendingImageEditReference }), clearPendingImageEditReference: () => set({ pendingImageEditReference: null }), - setContextUsage: (contextUsage) => set({ contextUsage }), + // Write through to the visible thread's own entry, so a value restored by the history loader + // survives a switch away and back: that loader runs once per mount and setActiveThreadId + // reads the map, so without this the bar goes blank on return. + setContextUsage: (contextUsage) => + set((state) => { + if (!state.activeThreadId) return { contextUsage }; + const next = { ...state.contextUsageByThreadId }; + if (contextUsage) { + next[state.activeThreadId] = contextUsage; + } else { + delete next[state.activeThreadId]; + } + return { contextUsage, contextUsageByThreadId: next }; + }), + setThreadContextUsage: (threadId, usage) => + set((state) => ({ + contextUsageByThreadId: { + ...state.contextUsageByThreadId, + [threadId]: usage, + }, + })), })); // Mirror token edits made through the shared store (e.g. Unsloth's field). diff --git a/studio/frontend/src/features/chat/stores/stop-running-chats-dialog-store.ts b/studio/frontend/src/features/chat/stores/stop-running-chats-dialog-store.ts new file mode 100644 index 0000000000..01ccc76f43 --- /dev/null +++ b/studio/frontend/src/features/chat/stores/stop-running-chats-dialog-store.ts @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { create } from "zustand"; + +type Resolver = (confirmed: boolean) => void; + +/** What confirming does to the model: reload it, or leave none loaded. */ +export type StopRunningChatsEffect = "reload" | "unload"; + +// One at a time: a new request declines any pending one so no promise leaks. +let pendingResolver: Resolver | null = null; + +interface StopRunningChatsDialogStore { + open: boolean; + /** How many conversations the pending action would stop. */ + count: number; + /** Titles of those conversations, when known, for the dialog body. */ + titles: string[]; + /** What the user is about to do, e.g. "Loading a different model". */ + action: string; + /** The set includes an embeddings/completions/audio request, which is not a chat. */ + hasNonChat: boolean; + /** Ejecting leaves no model loaded, so it must not be described as a reload. */ + effect: StopRunningChatsEffect; + requestConfirm: (args: { + count: number; + titles?: string[]; + action?: string; + hasNonChat?: boolean; + effect?: StopRunningChatsEffect; + }) => Promise; + resolve: (confirmed: boolean) => void; +} + +export const useStopRunningChatsDialogStore = + create()((set) => ({ + open: false, + count: 0, + titles: [], + action: "", + hasNonChat: false, + effect: "reload", + requestConfirm: ({ + count, + titles = [], + action = "", + hasNonChat = false, + effect = "reload", + }) => + new Promise((resolve) => { + pendingResolver?.(false); + pendingResolver = resolve; + set({ open: true, count, titles, action, hasNonChat, effect }); + }), + resolve: (confirmed) => { + const resolver = pendingResolver; + pendingResolver = null; + set({ + open: false, + count: 0, + titles: [], + action: "", + hasNonChat: false, + effect: "reload", + }); + resolver?.(confirmed); + }, + })); diff --git a/studio/frontend/src/features/chat/tool-approval.ts b/studio/frontend/src/features/chat/tool-approval.ts new file mode 100644 index 0000000000..b1908dacc0 --- /dev/null +++ b/studio/frontend/src/features/chat/tool-approval.ts @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"use client"; + +import { useChatRuntimeStore } from "./stores/chat-runtime-store"; + +/** + * True while this card's call is parked on the Allow / Deny prompt, so it can say it is + * waiting rather than counting up "Running". Set when the backend gates the call. + */ +export function useToolAwaitingApproval(toolCallId?: string): boolean { + return useChatRuntimeStore( + (s) => + !!toolCallId && + Object.prototype.hasOwnProperty.call(s.toolConfirmations, toolCallId), + ); +} diff --git a/studio/frontend/src/features/chat/tool-output-scope.ts b/studio/frontend/src/features/chat/tool-output-scope.ts index a7885a3916..abf8163431 100644 --- a/studio/frontend/src/features/chat/tool-output-scope.ts +++ b/studio/frontend/src/features/chat/tool-output-scope.ts @@ -3,6 +3,7 @@ "use client"; +import { useAuiState } from "@assistant-ui/react"; import { createContext, useContext } from "react"; import type { ModelType } from "./types"; @@ -20,10 +21,58 @@ export function toolPaneScope(modelType?: ModelType, pairId?: string): string { return `${modelType ?? "base"}\u0000${pairId ?? ""}`; } +/** + * Narrow a pane scope to one conversation: two threads in a pane can both be mid "call_0", + * so without the thread in the key they share a store entry and swap outputs. + */ +export function toolThreadScope(paneScope: string, threadId?: string): string { + return `${paneScope}\u0000${threadId ?? ""}`; +} + export const ToolPaneScopeContext = createContext(toolPaneScope()); +/** + * Store-key scope for the conversation this component renders in, taken from the surrounding + * runtime so reader and writer agree without a prop. + * + * `remoteId`, not `id`: the adapter gets `unstable_threadId`, which assistant-ui sources from + * `remoteId`, and an uninitialized thread has `id` but no `remoteId`. Reading `id` split the + * keys apart for the first turn of every New Chat, so live tool output never reached the card. + */ export function useToolPaneScope(): string { - return useContext(ToolPaneScopeContext); + const paneScope = useContext(ToolPaneScopeContext); + const threadId = useAuiState(({ threadListItem }) => threadListItem.remoteId); + return toolThreadScope(paneScope, threadId); +} + +/** + * Read a tool-output map for one call, tolerating a run that started before its thread had an id. + * + * The adapter captures its scope once at run start, so a first turn writes under the unresolved + * scope for its whole life. The autosave can assign `remoteId` mid-run, which moves this + * component's key but not the writer's, and the card went blank. Falling back to the pane-wide + * scope keeps those entries reachable; only an unpersisted first turn can be filed there. + */ +/** The scope a run that started before its thread had an id writes under. */ +export function useUnresolvedToolPaneScope(): string { + return toolThreadScope(useContext(ToolPaneScopeContext), undefined); +} + +export function useToolOutputFor( + map: Record, + paneScope: string, + toolCallId: string, +): string { + // Unconditional: hooks cannot sit behind the early return below. + const unresolvedScope = useUnresolvedToolPaneScope(); + // Only a thread mid-run can be the one that just gained its id. Local ids repeat + // ("call_0"), so an unconditional fallback showed a live first turn's stdout in every + // older conversation whose own entry had been cleared. + const isRunning = useAuiState(({ thread }) => thread.isRunning); + const own = map[toolOutputKey(paneScope, toolCallId)]; + if (own !== undefined) return own; + if (!isRunning) return ""; + return map[toolOutputKey(unresolvedScope, toolCallId)] ?? ""; } /** Store key for the live/full tool output maps: pane scope + tool call id. */ diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts index d554eb777e..3681b0f0cb 100644 --- a/studio/frontend/src/features/chat/types/api.ts +++ b/studio/frontend/src/features/chat/types/api.ts @@ -35,6 +35,11 @@ export interface ListLorasResponse { export interface LoadModelRequest { model_path: string; + /** + * Stop any chats still generating instead of getting a 409: a load replaces the single + * llama-server they all decode on. Set only after the user confirms. + */ + force_cancel_active?: boolean; nativePathLease?: string | null; hf_token: string | null; max_seq_length: number; @@ -201,6 +206,9 @@ export interface LoadModelResponse { export interface UnloadModelRequest { model_path: string; + /** Stop any chats still generating instead of getting a 409: the unload takes down the + * llama-server they all decode on. */ + force_cancel_active?: boolean; } export interface InferenceStatusResponse { diff --git a/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts b/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts new file mode 100644 index 0000000000..7b20ceb362 --- /dev/null +++ b/studio/frontend/src/features/chat/utils/confirm-stop-running-chats.ts @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { getActiveGenerations } from "../api/chat-api"; +import { useChatRuntimeStore } from "../stores/chat-runtime-store"; +import { + type StopRunningChatsEffect, + useStopRunningChatsDialogStore, +} from "../stores/stop-running-chats-dialog-store"; +import { listStoredChatThreads } from "./chat-history-storage"; + +export interface StopRunningChatsDecision { + /** False when the user chose to keep generating; the caller must not load. */ + proceed: boolean; + /** Pass as `force_cancel_active`. True only after an explicit confirmation, so the backend's 409 still guards every other caller. */ + forceCancelActive: boolean; +} + +/** + * Gate a model load / reload on the chats still generating: they share one llama-server, + * so a reload ends all of them. Ask first, then let the backend cancel them once the load + * is past preflight. External-provider chats are left out of both. + */ +export async function confirmStopRunningChatsIfNeeded( + action = "Loading a different model", + effect: StopRunningChatsEffect = "reload", +): Promise { + // Local runs only: an external-provider chat is not stopped by the swap, so counting it + // would block a safe load behind a dialog. The backend excludes them for the same reason. + const { runningByThreadId, localRunByThreadId } = + useChatRuntimeStore.getState(); + let running = Object.entries(runningByThreadId) + .filter(([threadId, on]) => on && localRunByThreadId[threadId]) + .map(([threadId]) => threadId); + let count = running.length; + let hasNonChat = false; + + // Always merge the backend snapshot: runningByThreadId is this tab's memory, empty after a + // reload and blind to a second tab, while force_cancel_active cancels every backend run. + // The union stays local-only, since external-provider runs are never in it. + try { + const active = await getActiveGenerations(); + const entries = active.active ?? []; + const merged = new Set(running); + for (const threadId of active.thread_ids ?? []) { + merged.add(threadId); + } + running = [...merged]; + // Count conversations, not handles: one chat holds several at once while a tool + // continuation registers its next leg before the previous unwinds, and active.count + // counts those separately. A first turn started before its id was persisted has no + // id to merge, so add those back or the prompt names fewer chats than will stop. + const unnamed = entries.filter((entry) => !entry.thread_id).length; + count = entries.length + ? running.length + unnamed + : Math.max(active.count ?? 0, running.length); + // Embeddings / completions / audio share the model but are not conversations, so the + // prompt must not offer to stop chats that do not exist. + hasNonChat = entries.some((entry) => (entry.kind ?? "chat") !== "chat"); + } catch { + // Backend unreachable / older build: fall back to the local map only. + } + + if (count === 0) { + return { proceed: true, forceCancelActive: false }; + } + + let titles: string[] = []; + try { + const threads = await listStoredChatThreads(); + const byId = new Map(threads.map((t) => [t.id, t])); + // A compare conversation runs two pane threads, and the sidebar and the route both treat + // it as one chat. Counting the raw ids asked to stop two and listed its title twice. Fold + // panes onto their pairId, keeping the backend's count when it is higher. + const seen = new Set(); + for (const id of running) { + const thread = byId.get(id); + const key = thread?.pairId ?? id; + if (seen.has(key)) continue; + seen.add(key); + titles.push(thread?.title || "Untitled chat"); + } + count = Math.max(seen.size, count - (running.length - seen.size)); + } catch { + // Titles are decoration; the count alone is enough to make the choice. + titles = []; + } + + const confirmed = await useStopRunningChatsDialogStore + .getState() + .requestConfirm({ count, titles, action, hasNonChat, effect }); + + if (!confirmed) { + return { proceed: false, forceCancelActive: false }; + } + + // Deliberately no local stop: the backend holds the cancel until the load clears preflight, + // so stopping now would truncate every chat even for a rejected load. + return { proceed: true, forceCancelActive: true }; +} diff --git a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts index ac1d973bfe..8013917ec8 100644 --- a/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts +++ b/studio/frontend/src/features/chat/utils/prompt-queue-boundary.ts @@ -1,8 +1,17 @@ export const PROMPT_QUEUE_STOP_EVENT = "unsloth:prompt-queue-stop"; -export function requestPromptQueueStop() { +export interface PromptQueueStopOptions { + /** Also cancel the prompt the queue already dispatched. Navigation passes `false` to + * leave it generating; an explicit stop passes `true` (the default). */ + cancelActiveRun?: boolean; +} + +export function requestPromptQueueStop(options: PromptQueueStopOptions = {}) { if (typeof window === "undefined") { return; } - window.dispatchEvent(new Event(PROMPT_QUEUE_STOP_EVENT)); + const { cancelActiveRun = true } = options; + window.dispatchEvent( + new CustomEvent(PROMPT_QUEUE_STOP_EVENT, { detail: { cancelActiveRun } }), + ); } diff --git a/studio/frontend/src/features/chat/utils/stop-chat-thread.ts b/studio/frontend/src/features/chat/utils/stop-chat-thread.ts new file mode 100644 index 0000000000..ba0fac9aa9 --- /dev/null +++ b/studio/frontend/src/features/chat/utils/stop-chat-thread.ts @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +import { useChatRuntimeStore } from "../stores/chat-runtime-store"; + +/** + * Stop one conversation's generation, visible or not. Returns true if a stop was dispatched. + * + * `cancelByThreadId` is assistant-ui's `cancelRun()`, registered only for the thread on screen; + * `serverCancelByThreadId` is registered for every run and POSTs that run's own `cancel_id`, so + * it is the only handle a background conversation has. Both are per-run. Runs with an unresolved + * thread id share the "__default" key, so stop every handle filed under it. + */ +export function stopChatThread(threadId: string | null | undefined): boolean { + if (!threadId) return false; + const { runningByThreadId, cancelByThreadId, serverCancelByThreadId } = + useChatRuntimeStore.getState(); + if (!runningByThreadId[threadId]) return false; + let stopped = false; + try { + const cancel = cancelByThreadId[threadId]; + if (cancel) { + cancel(); + stopped = true; + } + } catch { + // The run may have ended between the read above and this call. + } + // Also after cancelRun(): a proxy that swallows the fetch abort leaves the backend decoding. + for (const serverCancel of serverCancelByThreadId[threadId] ?? []) { + try { + serverCancel(); + stopped = true; + } catch { + // Same as above. + } + } + return stopped; +} diff --git a/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts b/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts index 2df9f712ff..7e1e1b0f5a 100644 --- a/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts +++ b/studio/frontend/src/features/transformers-upgrade/api/transformers-upgrade-api.ts @@ -16,14 +16,19 @@ interface InstallLatestTransformersResponse { latest_version?: string | null; } -/** Consented install of the latest transformers into the sidecar; synchronous, can take minutes. */ +/** Consented install of the latest transformers into the sidecar; synchronous, can take minutes. + * + * `forceCancelActive` carries the answer the user already gave the model swap's "stop N + * chats" prompt: without it the install 409s while those chats run, and nothing between the + * two dialogs stops them. Only ever true after that confirmation. */ export async function installLatestTransformers( version: string, + forceCancelActive = false, ): Promise { const response = await authFetch("/api/inference/install-latest-transformers", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ version }), + body: JSON.stringify({ version, force_cancel_active: forceCancelActive }), }); if (!response.ok) { throw new Error(await readFastApiError(response)); diff --git a/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts b/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts index 7d79d08d9c..9b942a7008 100644 --- a/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts +++ b/studio/frontend/src/features/transformers-upgrade/hooks/use-transformers-upgrade-consent.ts @@ -10,6 +10,9 @@ interface ConfirmArgs { upgrade: TransformersUpgradeInfo | null | undefined; /** When no release is installable, offer continuing into the caller's custom-code gate. */ trustRemoteCodeFallback?: boolean; + /** The caller already confirmed the swap's "stop N chats" prompt: carry it into + * the install, which otherwise 409s on those same chats with no way forward. */ + forceCancelActive?: boolean; } /** Pause a load needing a newer transformers on the consent dialog and run the install. @@ -18,11 +21,13 @@ export async function confirmTransformersUpgradeIfNeeded({ modelName, upgrade, trustRemoteCodeFallback, + forceCancelActive, }: ConfirmArgs): Promise { if (!upgrade) return true; return useTransformersUpgradeDialogStore .getState() .requestConsent(modelName, upgrade, { trustRemoteCodeFallback: Boolean(trustRemoteCodeFallback), + forceCancelActive: Boolean(forceCancelActive), }); } diff --git a/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts b/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts index 9e307fb1a1..be37691bc6 100644 --- a/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts +++ b/studio/frontend/src/features/transformers-upgrade/stores/transformers-upgrade-dialog-store.ts @@ -18,6 +18,9 @@ interface TransformersUpgradeDialogStore { errorMessage: string | null; /** Model ships custom code; without a PyPI install the load may fall back to trust_remote_code. */ trustRemoteCodeFallback: boolean; + /** The caller already confirmed the model swap's "stop N chats" prompt, so the install + * may stop them too; without it the install 409s and Retry can never succeed. */ + forceCancelActive: boolean; /** True once this consent's install completed. The install unloads the previous * model before swapping, so the caller must treat it as already unloaded; the * custom-code fallback resolves true without installing and leaves it loaded. */ @@ -34,7 +37,7 @@ interface TransformersUpgradeDialogStore { requestConsent: ( modelName: string, upgrade: TransformersUpgradeInfo, - options?: { trustRemoteCodeFallback?: boolean }, + options?: { trustRemoteCodeFallback?: boolean; forceCancelActive?: boolean }, ) => Promise; /** Accept/Retry: run the install; on success resolve(true) and close. */ install: () => Promise; @@ -49,6 +52,7 @@ export const useTransformersUpgradeDialogStore = phase: "consent", errorMessage: null, trustRemoteCodeFallback: false, + forceCancelActive: false, installRan: false, serverUnloadedChat: false, requestConsent: (modelName, upgrade, options) => @@ -62,6 +66,7 @@ export const useTransformersUpgradeDialogStore = phase: "consent", errorMessage: null, trustRemoteCodeFallback: Boolean(options?.trustRemoteCodeFallback), + forceCancelActive: Boolean(options?.forceCancelActive), installRan: false, }); }), @@ -71,14 +76,14 @@ export const useTransformersUpgradeDialogStore = return value; }, install: async () => { - const { upgrade, phase } = get(); + const { upgrade, phase, forceCancelActive } = get(); const version = upgrade?.pypi_version; if (!version || phase === "installing") return; const requestResolver = pendingResolver; set({ phase: "installing", errorMessage: null }); let result: Awaited>; try { - result = await installLatestTransformers(version); + result = await installLatestTransformers(version, forceCancelActive); // Latch the server-side unload IMMEDIATELY, before any resolver-identity // guard: even a superseded consent's install may have unloaded the chat // model, and the signal must survive for whichever load consumes it next. @@ -133,6 +138,7 @@ export const useTransformersUpgradeDialogStore = phase: "consent", errorMessage: null, trustRemoteCodeFallback: false, + forceCancelActive: false, }); resolver?.(installed); }, diff --git a/studio/frontend/src/i18n/locales/ar.ts b/studio/frontend/src/i18n/locales/ar.ts index 4b2a9e5ec0..47d5032fae 100644 --- a/studio/frontend/src/i18n/locales/ar.ts +++ b/studio/frontend/src/i18n/locales/ar.ts @@ -37,6 +37,8 @@ export const ar = { navigation: { newChat: "محادثة جديدة", returnToChat: "العودة إلى المحادثة", + returnToChats: "العودة إلى {count} محادثات", + chatGenerating: "جارٍ الإنشاء", compare: "مقارنة", search: "بحث", hub: "مركز النماذج", diff --git a/studio/frontend/src/i18n/locales/de.ts b/studio/frontend/src/i18n/locales/de.ts index a508fbb9fc..cb7d603f42 100644 --- a/studio/frontend/src/i18n/locales/de.ts +++ b/studio/frontend/src/i18n/locales/de.ts @@ -37,6 +37,8 @@ export const de = { navigation: { newChat: "Neuer Chat", returnToChat: "Zurück zum Chat", + returnToChats: "Zurück zu {count} Chats", + chatGenerating: "Wird generiert", compare: "Vergleichen", search: "Suchen", hub: "Modell-Hub", diff --git a/studio/frontend/src/i18n/locales/en.ts b/studio/frontend/src/i18n/locales/en.ts index 955a876dd5..bdfcf38231 100644 --- a/studio/frontend/src/i18n/locales/en.ts +++ b/studio/frontend/src/i18n/locales/en.ts @@ -34,6 +34,8 @@ export const en = { navigation: { newChat: "New chat", returnToChat: "Return to Chat", + returnToChats: "Return to {count} Chats", + chatGenerating: "Generating", compare: "Compare", search: "Search", hub: "Model hub", diff --git a/studio/frontend/src/i18n/locales/es.ts b/studio/frontend/src/i18n/locales/es.ts index 26e5e062dd..f7cb0e11f6 100644 --- a/studio/frontend/src/i18n/locales/es.ts +++ b/studio/frontend/src/i18n/locales/es.ts @@ -37,6 +37,8 @@ export const es = { navigation: { newChat: "Nuevo chat", returnToChat: "Volver al chat", + returnToChats: "Volver a {count} chats", + chatGenerating: "Generando", compare: "Comparar", search: "Buscar", hub: "Centro de modelos", diff --git a/studio/frontend/src/i18n/locales/fr.ts b/studio/frontend/src/i18n/locales/fr.ts index 704eac3fe2..4f2838391f 100644 --- a/studio/frontend/src/i18n/locales/fr.ts +++ b/studio/frontend/src/i18n/locales/fr.ts @@ -37,6 +37,8 @@ export const fr = { navigation: { newChat: "Nouvelle discussion", returnToChat: "Retour à la discussion", + returnToChats: "Retour à {count} discussions", + chatGenerating: "Génération en cours", compare: "Comparer", search: "Rechercher", hub: "Hub de modèles", diff --git a/studio/frontend/src/i18n/locales/hi.ts b/studio/frontend/src/i18n/locales/hi.ts index c18a86809f..33b827f314 100644 --- a/studio/frontend/src/i18n/locales/hi.ts +++ b/studio/frontend/src/i18n/locales/hi.ts @@ -37,6 +37,8 @@ export const hi = { navigation: { newChat: "नई चैट", returnToChat: "चैट पर लौटें", + returnToChats: "{count} चैट पर लौटें", + chatGenerating: "जनरेट हो रहा है", compare: "तुलना करें", search: "खोजें", hub: "मॉडल हब", diff --git a/studio/frontend/src/i18n/locales/ja.ts b/studio/frontend/src/i18n/locales/ja.ts index 9cde9c98ed..978fde6281 100644 --- a/studio/frontend/src/i18n/locales/ja.ts +++ b/studio/frontend/src/i18n/locales/ja.ts @@ -38,6 +38,8 @@ export const ja = { navigation: { newChat: "新規チャット", returnToChat: "チャットに戻る", + returnToChats: "{count} 件のチャットに戻る", + chatGenerating: "生成中", compare: "比較", search: "検索", hub: "モデルハブ", diff --git a/studio/frontend/src/i18n/locales/ko.ts b/studio/frontend/src/i18n/locales/ko.ts index a5b7c14940..aa8a4fd47b 100644 --- a/studio/frontend/src/i18n/locales/ko.ts +++ b/studio/frontend/src/i18n/locales/ko.ts @@ -37,6 +37,8 @@ export const ko = { navigation: { newChat: "새 채팅", returnToChat: "채팅으로 돌아가기", + returnToChats: "채팅 {count}개로 돌아가기", + chatGenerating: "생성 중", compare: "비교", search: "검색", hub: "모델 허브", diff --git a/studio/frontend/src/i18n/locales/pt-br.ts b/studio/frontend/src/i18n/locales/pt-br.ts index 5922e69890..84cd3f945e 100644 --- a/studio/frontend/src/i18n/locales/pt-br.ts +++ b/studio/frontend/src/i18n/locales/pt-br.ts @@ -37,6 +37,8 @@ export const ptBR = { navigation: { newChat: "Novo Chat", returnToChat: "Retornar ao Chat", + returnToChats: "Retornar a {count} chats", + chatGenerating: "Gerando", compare: "Comparar", search: "Buscar", hub: "Hub de modelos", diff --git a/studio/frontend/src/i18n/locales/ru.ts b/studio/frontend/src/i18n/locales/ru.ts index c680ff1ba9..7725212e3b 100644 --- a/studio/frontend/src/i18n/locales/ru.ts +++ b/studio/frontend/src/i18n/locales/ru.ts @@ -37,6 +37,8 @@ export const ru = { navigation: { newChat: "Новый чат", returnToChat: "Вернуться к чату", + returnToChats: "Вернуться к {count} чатам", + chatGenerating: "Генерация", compare: "Сравнить", search: "Поиск", hub: "Хаб моделей", diff --git a/studio/frontend/src/i18n/locales/zh-CN.ts b/studio/frontend/src/i18n/locales/zh-CN.ts index 22f1e06d80..06326ed008 100644 --- a/studio/frontend/src/i18n/locales/zh-CN.ts +++ b/studio/frontend/src/i18n/locales/zh-CN.ts @@ -37,6 +37,8 @@ export const zhCN = { navigation: { newChat: "新聊天", returnToChat: "返回聊天", + returnToChats: "返回 {count} 个聊天", + chatGenerating: "生成中", compare: "对比", search: "搜索", hub: "模型中心", diff --git a/tests/studio/test_cancel_atomicity.py b/tests/studio/test_cancel_atomicity.py index 391ef043d7..23f340f762 100644 --- a/tests/studio/test_cancel_atomicity.py +++ b/tests/studio/test_cancel_atomicity.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import importlib.util import random import threading from pathlib import Path @@ -107,6 +108,20 @@ _WANTED = { } +def _load_active_generations(): + """The real registry `_TrackedCancel` records runs in. + + Loaded straight off disk rather than imported, so the extracted class runs + against the genuine module without pulling in the whole route package (and + without putting studio/backend on sys.path for the rest of the session). + """ + path = SOURCE_PATH.parents[1] / "state" / "active_generations.py" + spec = importlib.util.spec_from_file_location("studio_active_generations", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _load_registry_module(): chunks = [] for n in _TREE.body: @@ -125,7 +140,7 @@ def _load_registry_module(): and n.target.id in _WANTED ): chunks.append(seg) - mod = {} + mod = {"active_generations": _load_active_generations()} exec( "import threading, time\nfrom typing import Optional\n" + "\n\n".join(chunks), mod, diff --git a/tests/studio/test_deep_research_frontend_contract.py b/tests/studio/test_deep_research_frontend_contract.py index b22d4691a1..c8636518cc 100644 --- a/tests/studio/test_deep_research_frontend_contract.py +++ b/tests/studio/test_deep_research_frontend_contract.py @@ -60,7 +60,9 @@ def test_research_mode_is_single_chat_and_detaches_without_cancel() -> None: assert 'update.event?.event === "reasoning.updated"' in adapter assert "The activity store coalesces these high-frequency events" in adapter assert '{ type: "text" as const, text: report }' in adapter - assert "if (abortSignal.aborted) return" in adapter + # runSignal, not abortSignal: each run gets its own controller, forwarded from the thread + # signal, so one chat's Stop cannot abort a sibling streaming in the background. + assert "if (runSignal.aborted) return" in adapter assert "await autoLoadSmallestModel()" in adapter assert "signal: researchFollowController.signal" in adapter assert "beginExternalResearchFollow(" in adapter diff --git a/tests/studio/test_first_turn_thread_identity.py b/tests/studio/test_first_turn_thread_identity.py new file mode 100644 index 0000000000..49dbc83ef9 --- /dev/null +++ b/tests/studio/test_first_turn_thread_identity.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""A first turn must reach the model adapter with its real thread id. + +assistant-ui binds `unstable_threadId` before the thread is persisted, so a first +turn used to file every run handle under the shared "__default" key. Two of them +overlapping there is unresolvable after the fact: nothing links a run under that +key to the id its thread later receives, so the sidebar showed no spinner and Stop +could not reach either generation. + +The link exists earlier. `append()` tracks `threadListItem.initialize()` by the +user message id, and `createPersistedRunAdapter` already awaits that promise before +invoking the adapter, so the id is known by the time the run starts. These tests pin +that the resolved id is carried through rather than discarded. +""" + +from __future__ import annotations + +import re +from pathlib import Path + + +WORKSPACE = Path(__file__).resolve().parents[2] +PROVIDER = (WORKSPACE / "studio/frontend/src/features/chat/runtime-provider.tsx").read_text( + encoding = "utf-8" +) + + +def test_the_tracked_promise_carries_the_assigned_thread_id(): + # Resolving to void threw the id away, which is what forced the "__default" detour. + assert "Promise\n>();" in PROVIDER + assert re.search( + r"trackRunStartReady\(\s*message\.id,\s*initializeThread\.then\(\(\{ remoteId \}\) => remoteId\),", + PROVIDER, + ), "append() must track the promise that resolves to the persisted thread id" + + +def test_wait_for_run_start_returns_the_id(): + assert re.search( + r"async function waitForRunStartHistoryAppend\([^)]*\): Promise", + PROVIDER, + re.S, + ), "the awaiter must hand back the id it waited for" + assert "return adoptedThreadId;" in PROVIDER + + +def test_the_run_is_given_its_real_thread_id(): + # The whole point: the adapter must not start under the unresolved key when the id is + # already known by the time the await above resolves. + block = re.search( + r"async \*run\(options\) \{.*?const result = adapter\.run\(.*?\);", + PROVIDER, + re.S, + ) + assert block, "createPersistedRunAdapter's run wrapper not found" + body = block.group(0) + assert "const adoptedThreadId = await waitForRunStartHistoryAppend(" in body + assert ( + "!options.unstable_threadId && adoptedThreadId" in body + ), "only fill in the id when assistant-ui had none" + assert "unstable_threadId: adoptedThreadId" in body + + +def test_an_existing_thread_id_is_never_overwritten(): + # A resolved thread already streams under its own id; replacing it would move a running + # chat's handles out from under the sidebar row watching them. + block = re.search( + r"const result = adapter\.run\((.*?)\);", + PROVIDER, + re.S, + ) + assert block + arg = block.group(1) + assert "? { ...options, unstable_threadId: adoptedThreadId }" in arg + assert ": options" in arg diff --git a/tests/studio/test_stop_running_chats_prompt_contract.py b/tests/studio/test_stop_running_chats_prompt_contract.py new file mode 100644 index 0000000000..d3995ec228 --- /dev/null +++ b/tests/studio/test_stop_running_chats_prompt_contract.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Source contracts for the "stop running chats" confirmation. + +The dialog is what a user reads before losing in-flight work, so two things have +to hold: it counts conversations rather than generation handles, and it describes +what confirming actually does. There is no frontend test runner in this repo, so +these read the source the way the other frontend contracts here do. +""" + +from __future__ import annotations + +from pathlib import Path + +WORKDIR = Path(__file__).resolve().parents[2] +FRONTEND = WORKDIR / "studio" / "frontend" / "src" + + +def _read(rel: str) -> str: + path = FRONTEND / rel + assert path.exists(), f"missing source file: {path}" + return path.read_text(encoding = "utf-8") + + +def test_the_prompt_counts_conversations_not_generation_handles(): + # One chat holds several handles while a tool continuation registers its next leg + # before the previous unwinds (active_generations.ActiveGeneration mints one per + # __enter__), so active.count exceeds the deduplicated thread_ids and the dialog + # offered to stop two chats while listing one title. + src = _read("features/chat/utils/confirm-stop-running-chats.ts") + assert "entry.thread_id" in src, "the unnamed entries have to be counted separately" + # The raw handle count survives only for a backend too old to send the entries. + primary = src.index("running.length + unnamed") + fallback = src.index("Math.max(active.count") + assert primary < fallback, "the handle count must be the fallback, not the primary" + + +def test_an_unload_is_not_described_as_a_reload(): + # ejectModel confirms through the same dialog, but confirming calls /unload and + # leaves no model loaded: "Unloading the model reloads the model" and "Stop and + # reload" promised the opposite for the destructive one. + dialog = _read("features/chat/components/stop-running-chats-dialog.tsx") + assert "Stop and unload" in dialog and "Stop and reload" in dialog + assert "leaves no model loaded" in dialog + + runtime = _read("features/chat/hooks/use-chat-model-runtime.ts") + eject = runtime.index('"Unloading the model"') + assert ( + '"unload"' in runtime[eject : eject + 120] + ), "the eject path must ask for the unload wording" + + +def test_the_tts_request_names_its_thread(): + # The audio branch registers its run locally under the thread key, and the backend + # tracker reads payload.thread_id. Omitting it filed the backend entry under no + # thread, so the prompt counted the named local run and the unnamed backend one as + # two requests for a single TTS chat. + src = _read("features/chat/api/chat-adapter.ts") + call = src.index("const result = await generateAudio(") + assert ( + "thread_id: resolvedThreadId" in src[call : call + 600] + ), "the TTS payload must carry the resolved thread id" diff --git a/tests/studio/test_stream_cancel_registration_timing.py b/tests/studio/test_stream_cancel_registration_timing.py index a833006873..5a28432aed 100644 --- a/tests/studio/test_stream_cancel_registration_timing.py +++ b/tests/studio/test_stream_cancel_registration_timing.py @@ -10,6 +10,7 @@ from __future__ import annotations import ast import asyncio +import importlib.util import json import threading import time @@ -256,6 +257,20 @@ _WANTED = { } +def _load_active_generations(): + """The real registry `_TrackedCancel` records runs in. + + Loaded straight off disk rather than imported, so the extracted class runs + against the genuine module without pulling in the whole route package (and + without putting studio/backend on sys.path for the rest of the session). + """ + path = SOURCE_PATH.parents[1] / "state" / "active_generations.py" + spec = importlib.util.spec_from_file_location("studio_active_generations", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def _load_registry_module(): chunks = [] for n in _TREE.body: @@ -274,7 +289,7 @@ def _load_registry_module(): and n.target.id in _WANTED ): chunks.append(seg) - mod = {} + mod = {"active_generations": _load_active_generations()} exec("import threading, time\n" + "\n\n".join(chunks), mod) return mod @@ -674,16 +689,18 @@ def test_generate_stream_cancels_backend_on_stream_cancelled_error(): body_src = "\n".join(ast.unparse(stmt) for stmt in sub.body) found_cancel_handler = ( "cancel_event.set()" in body_src - and "backend.reset_generation_state()" in body_src + and "backend.reset_generation_state(cancel_event)" in body_src and any(isinstance(stmt, ast.Raise) and stmt.exc is None for stmt in sub.body) ) if isinstance(sub, ast.Try) and sub.finalbody: final_src = "\n".join(ast.unparse(stmt) for stmt in sub.finalbody) - found_finally_cleanup = ( + # Accumulate: an existence claim, and the cleanup sits in a nested try whose + # own finally only unregisters the swap-gate entry. + found_finally_cleanup = found_finally_cleanup or ( "not completed" in final_src and "not cancel_event.is_set()" in final_src and "cancel_event.set()" in final_src - and "backend.reset_generation_state()" in final_src + and "backend.reset_generation_state(cancel_event)" in final_src and _awaits_to_thread_gen_close(sub) ) @@ -731,11 +748,11 @@ def test_stream_chunks_cancel_branch_resets_backend_state(): ): continue body_src = "\n".join(ast.unparse(s) for s in sub.body) - if "backend.reset_generation_state()" in body_src: + if "backend.reset_generation_state(cancel_event)" in body_src: return raise AssertionError( "stream_chunks `if cancel_event.is_set():` branch must call " - "backend.reset_generation_state() -- matches the existing " + "backend.reset_generation_state(cancel_event) -- matches the existing " "request.is_disconnected() / CancelledError cleanup paths and " "prevents KV-cache drift after cancel-via-POST" ) diff --git a/unsloth_cli/commands/studio.py b/unsloth_cli/commands/studio.py index c2bdbbc915..864941a20a 100644 --- a/unsloth_cli/commands/studio.py +++ b/unsloth_cli/commands/studio.py @@ -305,7 +305,9 @@ def _find_setup_script() -> Optional[Path]: _PARALLEL_MIN = 1 _PARALLEL_MAX = 64 _PARALLEL_DEFAULT_RUN = 4 # pre-PR hardcoded for `unsloth studio run` -_PARALLEL_DEFAULT_PLAIN = 1 # pre-PR effective for plain `unsloth studio` +# New Chat leaves the previous conversation generating and the admission queue caps decodes at +# the slot count, so at 1 every extra chat queues. _slots_that_fit_on_gpu() may cut it back. +_PARALLEL_DEFAULT_PLAIN = 4 def _resolve_secure(secure: bool, not_secure: bool) -> bool: @@ -1261,8 +1263,7 @@ def studio_default( max = _PARALLEL_MAX, help = ( f"llama-server parallel decode slots ({_PARALLEL_MIN}..{_PARALLEL_MAX}). " - f"Default {_PARALLEL_DEFAULT_PLAIN}; `unsloth studio run` " - f"defaults to {_PARALLEL_DEFAULT_RUN}." + f"Default {_PARALLEL_DEFAULT_PLAIN}." ), ), cloudflare: Optional[bool] = typer.Option(