From ca1a801300a8a764a1012e63e40deb8a798603e0 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 9 Apr 2026 01:45:16 +0000 Subject: [PATCH] feat: explicit enable/disable toggle for API endpoint with --parallel restart - Add set_parallel(n) to LlamaCppBackend: kills and relaunches llama-server with updated --parallel flag (stores launch cmd/env for replay) - Add POST /access-endpoint/enable: restarts with --parallel 2, generates key - Add POST /access-endpoint/disable: restarts with --parallel 1, clears key - GET /access-endpoint now returns enabled state - Model load clears previous API key (user must explicitly enable) - Frontend dialog shows enable/disable toggle with loading state - Connection details (URLs, key, snippets) only shown when enabled --- studio/backend/core/inference/llama_cpp.py | 56 ++++ studio/backend/routes/inference.py | 95 +++++-- .../frontend/src/features/chat/chat-page.tsx | 239 ++++++++++++------ 3 files changed, 282 insertions(+), 108 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c84ac640df..a2947f8770 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -129,6 +129,8 @@ class LlamaCppBackend: self._stdout_thread: Optional[threading.Thread] = None self._cancel_event = threading.Event() self._api_key: Optional[str] = None + self._launch_cmd: Optional[list[str]] = None + self._launch_env: Optional[dict] = None self._kill_orphaned_servers() atexit.register(self._cleanup) @@ -1516,6 +1518,8 @@ class LlamaCppBackend: if gpu_indices is not None: env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_indices) + self._launch_cmd = list(cmd) + self._launch_env = dict(env) self._stdout_lines = [] self._process = subprocess.Popen( cmd, @@ -1631,6 +1635,58 @@ class LlamaCppBackend: torch.cuda.empty_cache() return True + def set_parallel(self, n: int) -> bool: + """Restart llama-server with a different --parallel value. + + Kills the running process and relaunches with the same command + but ``--parallel`` swapped. Returns True if the restarted server + passes the health check. + """ + if not self._launch_cmd or not self._launch_env: + raise RuntimeError("No launch command stored — cannot restart") + + cmd = list(self._launch_cmd) + # Swap --parallel value + try: + idx = cmd.index("--parallel") + cmd[idx + 1] = str(n) + except (ValueError, IndexError): + cmd.extend(["--parallel", str(n)]) + + with self._lock: + self._kill_process() + self._port = self._find_free_port() + # Update port in cmd + try: + pi = cmd.index("--port") + cmd[pi + 1] = str(self._port) + except (ValueError, IndexError): + cmd.extend(["--port", str(self._port)]) + + self._launch_cmd = list(cmd) + self._stdout_lines = [] + self._process = subprocess.Popen( + cmd, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + env = self._launch_env, + ) + self._stdout_thread = threading.Thread( + target = self._drain_stdout, daemon = True, name = "llama-stdout" + ) + self._stdout_thread.start() + + if not self._wait_for_health(timeout = 120.0): + self._kill_process() + raise RuntimeError("llama-server failed to restart with new --parallel") + + self._healthy = True + logger.info( + f"llama-server restarted with --parallel {n} on port {self._port}" + ) + return True + def _kill_process(self): """Terminate the subprocess if running.""" if self._process is None: diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0b72a636b8..77a7d6fbfa 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -303,10 +303,8 @@ async def load_model( inference_config = load_inference_config(config.identifier) - # Auto-generate API key for external OpenAI-compatible access - fastapi_request.app.state.external_api_key = ( - f"sk-unsloth-{_secrets.token_urlsafe(32)}" - ) + # Clear any previous API key — user must explicitly enable endpoint + fastapi_request.app.state.external_api_key = None return LoadResponse( status = "loaded", @@ -446,10 +444,8 @@ async def load_model( except Exception: pass - # Auto-generate API key for external OpenAI-compatible access - fastapi_request.app.state.external_api_key = ( - f"sk-unsloth-{_secrets.token_urlsafe(32)}" - ) + # Clear any previous API key — user must explicitly enable endpoint + fastapi_request.app.state.external_api_key = None return LoadResponse( status = "loaded", @@ -902,20 +898,11 @@ def _extract_content_parts( # ── Access Endpoint (external OpenAI-compatible access) ────────── -@router.get("/access-endpoint") -async def get_access_endpoint( - request: Request, - current_subject: str = Depends(get_current_subject), -): - """Return API key, local/external URLs, and model info for the Access Endpoint dialog.""" - api_key = getattr(request.app.state, "external_api_key", None) - if not api_key: - raise HTTPException(status_code = 400, detail = "No model loaded") - +def _build_endpoint_urls(request: Request): + """Return (local_url, external_url) for the Access Endpoint dialog.""" port = getattr(request.app.state, "server_port", None) local_url = f"http://127.0.0.1:{port}/v1" if port else f"{request.base_url}v1" - # Resolve external IP when bound to all interfaces (same as startup banner) external_url = None bind_host = getattr(request.app.state, "bind_host", None) if bind_host in ("0.0.0.0", "::") and port: @@ -925,22 +912,80 @@ async def get_access_endpoint( if ext_ip and ext_ip not in ("127.0.0.1", "0.0.0.0", "localhost"): external_url = f"http://{ext_ip}:{port}/v1" + return local_url, external_url + + +def _get_loaded_model_name(): + """Return the currently loaded model identifier.""" llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded: + return llama_backend.model_identifier backend = get_inference_backend() - model = ( - llama_backend.model_identifier - if llama_backend.is_loaded - else (list(backend.models.keys())[0] if backend.models else "default") - ) + if backend.models: + return list(backend.models.keys())[0] + return "default" + + +@router.get("/access-endpoint") +async def get_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Return endpoint state, API key, URLs, and model info.""" + api_key = getattr(request.app.state, "external_api_key", None) + local_url, external_url = _build_endpoint_urls(request) return { + "enabled": api_key is not None, "api_key": api_key, "local_url": local_url, "external_url": external_url, - "model": model, + "model": _get_loaded_model_name(), } +@router.post("/access-endpoint/enable") +async def enable_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Enable external API access: restart llama-server with --parallel 2 and generate an API key.""" + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException(status_code = 400, detail = "No GGUF model loaded") + + # Restart with --parallel 2 for concurrent Studio + external access + await asyncio.to_thread(llama_backend.set_parallel, 2) + + # Generate API key + api_key = f"sk-unsloth-{_secrets.token_urlsafe(32)}" + request.app.state.external_api_key = api_key + + local_url, external_url = _build_endpoint_urls(request) + return { + "enabled": True, + "api_key": api_key, + "local_url": local_url, + "external_url": external_url, + "model": _get_loaded_model_name(), + } + + +@router.post("/access-endpoint/disable") +async def disable_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Disable external API access: restart llama-server with --parallel 1 and clear the API key.""" + request.app.state.external_api_key = None + + llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded: + await asyncio.to_thread(llama_backend.set_parallel, 1) + + return {"enabled": False} + + @router.post("/chat/completions") async def openai_chat_completions( payload: ChatCompletionRequest, diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 55ddabc1dc..77c76ebb06 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -537,6 +537,8 @@ export function ChatPage(): ReactElement { const [view, setView] = useState(getInitialSingleChatView); const [settingsOpen, setSettingsOpen] = useState(false); const [endpointDialogOpen, setEndpointDialogOpen] = useState(false); + const [endpointEnabled, setEndpointEnabled] = useState(false); + const [endpointLoading, setEndpointLoading] = useState(false); const [endpointLocalUrl, setEndpointLocalUrl] = useState(""); const [endpointExternalUrl, setEndpointExternalUrl] = useState(null); const [endpointBaseUrl, setEndpointBaseUrl] = useState(""); @@ -609,6 +611,23 @@ print(completion.choices[0].message.content)`, [endpointApiKey, endpointBaseUrl, modelAlias], ); + const applyEndpointData = useCallback( + (data: { + enabled: boolean; + api_key: string | null; + local_url: string; + external_url: string | null; + model: string; + }) => { + setEndpointEnabled(data.enabled); + setEndpointApiKey(data.api_key ?? ""); + setEndpointLocalUrl(data.local_url); + setEndpointExternalUrl(data.external_url ?? null); + setEndpointBaseUrl(data.local_url); + }, + [], + ); + useEffect(() => { if (!endpointDialogOpen) return; let cancelled = false; @@ -616,16 +635,39 @@ print(completion.choices[0].message.content)`, .then((res) => (res.ok ? res.json() : null)) .then((data) => { if (cancelled || !data) return; - setEndpointApiKey(data.api_key); - setEndpointLocalUrl(data.local_url); - setEndpointExternalUrl(data.external_url ?? null); - setEndpointBaseUrl(data.local_url); + applyEndpointData(data); }) .catch(() => {}); return () => { cancelled = true; }; - }, [endpointDialogOpen]); + }, [endpointDialogOpen, applyEndpointData]); + + const handleToggleEndpoint = useCallback(async () => { + setEndpointLoading(true); + try { + const action = endpointEnabled ? "disable" : "enable"; + const res = await authFetch(`/api/inference/access-endpoint/${action}`, { + method: "POST", + }); + if (!res.ok) { + const err = await res.json().catch(() => null); + toast.error(err?.detail ?? "Failed to toggle endpoint"); + return; + } + const data = await res.json(); + applyEndpointData(data); + toast.success( + data.enabled + ? "API endpoint enabled" + : "API endpoint disabled", + ); + } catch { + toast.error("Failed to toggle endpoint"); + } finally { + setEndpointLoading(false); + } + }, [endpointEnabled, applyEndpointData]); const handleCheckpointChange = useCallback( ( @@ -1092,93 +1134,124 @@ print(completion.choices[0].message.content)`, Access Endpoint - Use these OpenAI-compatible settings to connect to the active - model from your own code. + Serve the active model as an OpenAI-compatible API endpoint + for use from your own code, scripts, or other applications. -
-
- -
- - {endpointBaseUrl !== endpointLocalUrl && ( - - )} -
+
+
+ {endpointEnabled ? ( + + Endpoint is running + + ) : ( + + Endpoint is off + + )}
- {endpointExternalUrl && ( -
- -
+ +
+ {endpointEnabled && ( + <> +
+
+ +
+ + {endpointBaseUrl !== endpointLocalUrl && ( + + )} +
+
+ {endpointExternalUrl && ( +
+ +
+ + {endpointBaseUrl !== endpointExternalUrl && ( + + )} +
+
+ )} +
+ +
+
+ + - {endpointBaseUrl !== endpointExternalUrl && ( - - )}
- )} -
- - -
-
- - -
-
-
-
- - Python (OpenAI SDK) - - -
-
- - cURL - - -
-
+
+
+ + Python (OpenAI SDK) + + +
+
+ + cURL + + +
+
+ + )}