ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: make Stop interrupt a llama.cpp generation stalled mid-stream (#7117)

Browse source 
* Studio: make Stop interrupt a llama.cpp generation stalled mid-stream

* Studio: tighten stream-cancel comments

---------

Co-authored-by: danielhanchen <danielhanchen@users.noreply.github.com>
This commit is contained in:
oobabooga 2026-07-14 09:11:56 -03:00 • committed by GitHub
commit 5de668926c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 116 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
studio/backend/core/inference/llama_cpp.py
View file

@ -8606,7 +8606,8 @@ class LlamaCppBackend:
@staticmethod
def _shutdown_active_httpx_sockets(client: "httpx.Client") -> None:
"""Best-effort interrupt for a sync httpx request blocked before headers."""
"""Best-effort interrupt for a sync httpx read blocked in recv(), whether
parked before headers (prefill) or mid-stream."""
try:
pool = getattr(getattr(client, "_transport", None), "_pool", None)
connections = list(getattr(pool, "_connections", []) or [])
@ -8654,10 +8655,11 @@ class LlamaCppBackend:
while not _cancel_closed.is_set():
r = _response_ref[0]
try:
# response.close() can't wake a read already blocked in
# recv(); only a socket shutdown does, so shut down first.
LlamaCppBackend._shutdown_active_httpx_sockets(client)
if r is not None:
r.close()
else:
LlamaCppBackend._shutdown_active_httpx_sockets(client)
return
except Exception as e:
logger.debug(f"Error closing request in cancel watcher: {e}")