From 43a633e550522c1ddc9142001d65a4e9664b3f23 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 9 Apr 2026 14:56:01 +0000 Subject: [PATCH] feat: boot llama-server with --parallel 4 by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise the default parallel slot count so Studio chat and the external API endpoint can run concurrently without a subprocess restart. Enable/disable of the Access Endpoint is now instant — it only mints or clears the API key, since the parallel slots are already in place from the initial load. --- studio/backend/core/inference/llama_cpp.py | 2 +- studio/backend/routes/inference.py | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index a2947f8770..8008f27b66 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1285,7 +1285,7 @@ class LlamaCppBackend: "-c", str(effective_ctx) if effective_ctx > 0 else "0", "--parallel", - "1", # Single-user studio, saves VRAM + "4", # Match LM Studio default: supports concurrent Studio chat + external API access "--flash-attn", "on", # Force flash attention for speed ] diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 77a7d6fbfa..a5957f414e 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -949,15 +949,13 @@ async def enable_access_endpoint( request: Request, current_subject: str = Depends(get_current_subject), ): - """Enable external API access: restart llama-server with --parallel 2 and generate an API key.""" + """Enable external API access: generate an API key for the running llama-server.""" llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: raise HTTPException(status_code = 400, detail = "No GGUF model loaded") - # Restart with --parallel 2 for concurrent Studio + external access - await asyncio.to_thread(llama_backend.set_parallel, 2) - - # Generate API key + # llama-server is already running with --parallel 4, so concurrent Studio + # chat + external API is supported without a restart. We just mint a key. api_key = f"sk-unsloth-{_secrets.token_urlsafe(32)}" request.app.state.external_api_key = api_key @@ -976,13 +974,8 @@ async def disable_access_endpoint( request: Request, current_subject: str = Depends(get_current_subject), ): - """Disable external API access: restart llama-server with --parallel 1 and clear the API key.""" + """Disable external API access: clear the API key. llama-server keeps running with --parallel 4.""" request.app.state.external_api_key = None - - llama_backend = get_llama_cpp_backend() - if llama_backend.is_loaded: - await asyncio.to_thread(llama_backend.set_parallel, 1) - return {"enabled": False}