From 228af61155e581a2b12909850940e98ec0dbec2f Mon Sep 17 00:00:00 2001 From: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Date: Wed, 8 Apr 2026 12:07:18 +0100 Subject: [PATCH 01/18] feat: frontend implementation for Access Endpoint (#4914) Co-authored-by: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> --- .../frontend/src/features/chat/chat-page.tsx | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index cf1ba11d7b..5d1c31d9e6 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -8,6 +8,14 @@ import { } from "@/components/assistant-ui/model-selector"; import { Thread } from "@/components/assistant-ui/thread"; import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; import { Sheet, SheetContent, @@ -33,6 +41,7 @@ import { Settings04Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; +import { code as codePlugin } from "@streamdown/code"; import { type CSSProperties, type ReactElement, @@ -44,6 +53,7 @@ import { useRef, useState, } from "react"; +import { Streamdown } from "streamdown"; import { toast } from "sonner"; import { listLocalModels } from "./api/chat-api"; import { ChatSettingsPanel } from "./chat-settings-sheet"; @@ -73,6 +83,42 @@ type LoraCandidate = { updatedAt?: number; }; +const ENDPOINT_BASE_URL_FALLBACK = "http://127.0.0.1:8001/v1"; +const SHIKI_THEME = ["github-light", "github-dark"] as ["github-light", "github-dark"]; + +function buildDefaultEndpointBaseUrl(): string { + if (typeof window === "undefined") { + return ENDPOINT_BASE_URL_FALLBACK; + } + return `${window.location.origin}/v1`; +} + +function HighlightedSnippet({ + language, + source, +}: { + language: "python" | "bash"; + source: string; +}) { + const markdown = useMemo( + () => `\`\`\`${language}\n${source}\n\`\`\``, + [language, source], + ); + + return ( +
+ + {markdown} + +
+ ); +} + function normalizeModelRef(value: string | null | undefined): string { return value?.trim().toLowerCase() ?? ""; } @@ -497,6 +543,11 @@ export function ChatPage(): ReactElement { // explicitly sets a nonce in handleNewThread. const [view, setView] = useState(getInitialSingleChatView); const [settingsOpen, setSettingsOpen] = useState(false); + const [endpointDialogOpen, setEndpointDialogOpen] = useState(false); + const [endpointBaseUrl, setEndpointBaseUrl] = useState( + buildDefaultEndpointBaseUrl, + ); + const [endpointApiKey, setEndpointApiKey] = useState("sk-no-key-required"); const [modelSelectorOpen, setModelSelectorOpen] = useState(false); const [modelSelectorLocked, setModelSelectorLocked] = useState(false); const [sidebarOpen, setSidebarOpen] = useState(true); @@ -537,6 +588,47 @@ export function ChatPage(): ReactElement { const canCompare = useMemo(() => { return Boolean(inferenceParams.checkpoint); }, [inferenceParams.checkpoint]); + const modelAlias = inferenceParams.checkpoint || "unsloth/your-model-alias"; + const endpointPythonSnippet = useMemo( + () => `from openai import OpenAI + +client = OpenAI( + base_url="${endpointBaseUrl}", + api_key="${endpointApiKey}", +) + +completion = client.chat.completions.create( + model="${modelAlias}", + messages=[{"role": "user", "content": "What is 2+2?"}], +) + +print(completion.choices[0].message.content)`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const endpointCurlSnippet = useMemo( + () => `curl ${endpointBaseUrl}/chat/completions \\ + -H "Content-Type: application/json" \\ + -H "Authorization: Bearer ${endpointApiKey}" \\ + -d '{ + "model": "${modelAlias}", + "messages": [{"role": "user", "content": "What is 2+2?"}] + }'`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + + useEffect(() => { + if (typeof window === "undefined") return; + setEndpointBaseUrl((prev) => + prev === ENDPOINT_BASE_URL_FALLBACK ? buildDefaultEndpointBaseUrl() : prev, + ); + }, []); + + useEffect(() => { + if (!endpointDialogOpen) return; + setEndpointBaseUrl((prev) => + prev.trim().length === 0 ? buildDefaultEndpointBaseUrl() : prev, + ); + }, [endpointDialogOpen]); const handleCheckpointChange = useCallback( ( @@ -641,6 +733,13 @@ export function ChatPage(): ReactElement { const openSettings = useCallback(() => setSettingsOpen(true), []); const closeSettings = useCallback(() => setSettingsOpen(false), []); const openSidebar = useCallback(() => setSidebarOpen(true), []); + const handleOpenEndpointDialog = useCallback(() => { + if (!inferenceParams.checkpoint) { + toast.message("Load a model first to access endpoint details."); + return; + } + setEndpointDialogOpen(true); + }, [inferenceParams.checkpoint]); const enterCompare = useCallback(() => { setViewBeforeCompare((prev) => prev ?? view); @@ -932,6 +1031,17 @@ export function ChatPage(): ReactElement { completionTokens={contextUsage.completionTokens} /> ) : null} + {inferenceParams.checkpoint ? ( + + ) : null} + )} + + {endpointExternalUrl && ( +
+ +
+ + {endpointBaseUrl !== endpointExternalUrl && ( + + )} +
+
+ )}
From ca1a801300a8a764a1012e63e40deb8a798603e0 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 9 Apr 2026 01:45:16 +0000 Subject: [PATCH 03/18] feat: explicit enable/disable toggle for API endpoint with --parallel restart - Add set_parallel(n) to LlamaCppBackend: kills and relaunches llama-server with updated --parallel flag (stores launch cmd/env for replay) - Add POST /access-endpoint/enable: restarts with --parallel 2, generates key - Add POST /access-endpoint/disable: restarts with --parallel 1, clears key - GET /access-endpoint now returns enabled state - Model load clears previous API key (user must explicitly enable) - Frontend dialog shows enable/disable toggle with loading state - Connection details (URLs, key, snippets) only shown when enabled --- studio/backend/core/inference/llama_cpp.py | 56 ++++ studio/backend/routes/inference.py | 95 +++++-- .../frontend/src/features/chat/chat-page.tsx | 239 ++++++++++++------ 3 files changed, 282 insertions(+), 108 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c84ac640df..a2947f8770 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -129,6 +129,8 @@ class LlamaCppBackend: self._stdout_thread: Optional[threading.Thread] = None self._cancel_event = threading.Event() self._api_key: Optional[str] = None + self._launch_cmd: Optional[list[str]] = None + self._launch_env: Optional[dict] = None self._kill_orphaned_servers() atexit.register(self._cleanup) @@ -1516,6 +1518,8 @@ class LlamaCppBackend: if gpu_indices is not None: env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_indices) + self._launch_cmd = list(cmd) + self._launch_env = dict(env) self._stdout_lines = [] self._process = subprocess.Popen( cmd, @@ -1631,6 +1635,58 @@ class LlamaCppBackend: torch.cuda.empty_cache() return True + def set_parallel(self, n: int) -> bool: + """Restart llama-server with a different --parallel value. + + Kills the running process and relaunches with the same command + but ``--parallel`` swapped. Returns True if the restarted server + passes the health check. + """ + if not self._launch_cmd or not self._launch_env: + raise RuntimeError("No launch command stored — cannot restart") + + cmd = list(self._launch_cmd) + # Swap --parallel value + try: + idx = cmd.index("--parallel") + cmd[idx + 1] = str(n) + except (ValueError, IndexError): + cmd.extend(["--parallel", str(n)]) + + with self._lock: + self._kill_process() + self._port = self._find_free_port() + # Update port in cmd + try: + pi = cmd.index("--port") + cmd[pi + 1] = str(self._port) + except (ValueError, IndexError): + cmd.extend(["--port", str(self._port)]) + + self._launch_cmd = list(cmd) + self._stdout_lines = [] + self._process = subprocess.Popen( + cmd, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + env = self._launch_env, + ) + self._stdout_thread = threading.Thread( + target = self._drain_stdout, daemon = True, name = "llama-stdout" + ) + self._stdout_thread.start() + + if not self._wait_for_health(timeout = 120.0): + self._kill_process() + raise RuntimeError("llama-server failed to restart with new --parallel") + + self._healthy = True + logger.info( + f"llama-server restarted with --parallel {n} on port {self._port}" + ) + return True + def _kill_process(self): """Terminate the subprocess if running.""" if self._process is None: diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0b72a636b8..77a7d6fbfa 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -303,10 +303,8 @@ async def load_model( inference_config = load_inference_config(config.identifier) - # Auto-generate API key for external OpenAI-compatible access - fastapi_request.app.state.external_api_key = ( - f"sk-unsloth-{_secrets.token_urlsafe(32)}" - ) + # Clear any previous API key — user must explicitly enable endpoint + fastapi_request.app.state.external_api_key = None return LoadResponse( status = "loaded", @@ -446,10 +444,8 @@ async def load_model( except Exception: pass - # Auto-generate API key for external OpenAI-compatible access - fastapi_request.app.state.external_api_key = ( - f"sk-unsloth-{_secrets.token_urlsafe(32)}" - ) + # Clear any previous API key — user must explicitly enable endpoint + fastapi_request.app.state.external_api_key = None return LoadResponse( status = "loaded", @@ -902,20 +898,11 @@ def _extract_content_parts( # ── Access Endpoint (external OpenAI-compatible access) ────────── -@router.get("/access-endpoint") -async def get_access_endpoint( - request: Request, - current_subject: str = Depends(get_current_subject), -): - """Return API key, local/external URLs, and model info for the Access Endpoint dialog.""" - api_key = getattr(request.app.state, "external_api_key", None) - if not api_key: - raise HTTPException(status_code = 400, detail = "No model loaded") - +def _build_endpoint_urls(request: Request): + """Return (local_url, external_url) for the Access Endpoint dialog.""" port = getattr(request.app.state, "server_port", None) local_url = f"http://127.0.0.1:{port}/v1" if port else f"{request.base_url}v1" - # Resolve external IP when bound to all interfaces (same as startup banner) external_url = None bind_host = getattr(request.app.state, "bind_host", None) if bind_host in ("0.0.0.0", "::") and port: @@ -925,22 +912,80 @@ async def get_access_endpoint( if ext_ip and ext_ip not in ("127.0.0.1", "0.0.0.0", "localhost"): external_url = f"http://{ext_ip}:{port}/v1" + return local_url, external_url + + +def _get_loaded_model_name(): + """Return the currently loaded model identifier.""" llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded: + return llama_backend.model_identifier backend = get_inference_backend() - model = ( - llama_backend.model_identifier - if llama_backend.is_loaded - else (list(backend.models.keys())[0] if backend.models else "default") - ) + if backend.models: + return list(backend.models.keys())[0] + return "default" + + +@router.get("/access-endpoint") +async def get_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Return endpoint state, API key, URLs, and model info.""" + api_key = getattr(request.app.state, "external_api_key", None) + local_url, external_url = _build_endpoint_urls(request) return { + "enabled": api_key is not None, "api_key": api_key, "local_url": local_url, "external_url": external_url, - "model": model, + "model": _get_loaded_model_name(), } +@router.post("/access-endpoint/enable") +async def enable_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Enable external API access: restart llama-server with --parallel 2 and generate an API key.""" + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException(status_code = 400, detail = "No GGUF model loaded") + + # Restart with --parallel 2 for concurrent Studio + external access + await asyncio.to_thread(llama_backend.set_parallel, 2) + + # Generate API key + api_key = f"sk-unsloth-{_secrets.token_urlsafe(32)}" + request.app.state.external_api_key = api_key + + local_url, external_url = _build_endpoint_urls(request) + return { + "enabled": True, + "api_key": api_key, + "local_url": local_url, + "external_url": external_url, + "model": _get_loaded_model_name(), + } + + +@router.post("/access-endpoint/disable") +async def disable_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Disable external API access: restart llama-server with --parallel 1 and clear the API key.""" + request.app.state.external_api_key = None + + llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded: + await asyncio.to_thread(llama_backend.set_parallel, 1) + + return {"enabled": False} + + @router.post("/chat/completions") async def openai_chat_completions( payload: ChatCompletionRequest, diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 55ddabc1dc..77c76ebb06 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -537,6 +537,8 @@ export function ChatPage(): ReactElement { const [view, setView] = useState(getInitialSingleChatView); const [settingsOpen, setSettingsOpen] = useState(false); const [endpointDialogOpen, setEndpointDialogOpen] = useState(false); + const [endpointEnabled, setEndpointEnabled] = useState(false); + const [endpointLoading, setEndpointLoading] = useState(false); const [endpointLocalUrl, setEndpointLocalUrl] = useState(""); const [endpointExternalUrl, setEndpointExternalUrl] = useState(null); const [endpointBaseUrl, setEndpointBaseUrl] = useState(""); @@ -609,6 +611,23 @@ print(completion.choices[0].message.content)`, [endpointApiKey, endpointBaseUrl, modelAlias], ); + const applyEndpointData = useCallback( + (data: { + enabled: boolean; + api_key: string | null; + local_url: string; + external_url: string | null; + model: string; + }) => { + setEndpointEnabled(data.enabled); + setEndpointApiKey(data.api_key ?? ""); + setEndpointLocalUrl(data.local_url); + setEndpointExternalUrl(data.external_url ?? null); + setEndpointBaseUrl(data.local_url); + }, + [], + ); + useEffect(() => { if (!endpointDialogOpen) return; let cancelled = false; @@ -616,16 +635,39 @@ print(completion.choices[0].message.content)`, .then((res) => (res.ok ? res.json() : null)) .then((data) => { if (cancelled || !data) return; - setEndpointApiKey(data.api_key); - setEndpointLocalUrl(data.local_url); - setEndpointExternalUrl(data.external_url ?? null); - setEndpointBaseUrl(data.local_url); + applyEndpointData(data); }) .catch(() => {}); return () => { cancelled = true; }; - }, [endpointDialogOpen]); + }, [endpointDialogOpen, applyEndpointData]); + + const handleToggleEndpoint = useCallback(async () => { + setEndpointLoading(true); + try { + const action = endpointEnabled ? "disable" : "enable"; + const res = await authFetch(`/api/inference/access-endpoint/${action}`, { + method: "POST", + }); + if (!res.ok) { + const err = await res.json().catch(() => null); + toast.error(err?.detail ?? "Failed to toggle endpoint"); + return; + } + const data = await res.json(); + applyEndpointData(data); + toast.success( + data.enabled + ? "API endpoint enabled" + : "API endpoint disabled", + ); + } catch { + toast.error("Failed to toggle endpoint"); + } finally { + setEndpointLoading(false); + } + }, [endpointEnabled, applyEndpointData]); const handleCheckpointChange = useCallback( ( @@ -1092,93 +1134,124 @@ print(completion.choices[0].message.content)`, Access Endpoint - Use these OpenAI-compatible settings to connect to the active - model from your own code. + Serve the active model as an OpenAI-compatible API endpoint + for use from your own code, scripts, or other applications. -
-
- -
- - {endpointBaseUrl !== endpointLocalUrl && ( - - )} -
+
+
+ {endpointEnabled ? ( + + Endpoint is running + + ) : ( + + Endpoint is off + + )}
- {endpointExternalUrl && ( -
- -
+ +
+ {endpointEnabled && ( + <> +
+
+ +
+ + {endpointBaseUrl !== endpointLocalUrl && ( + + )} +
+
+ {endpointExternalUrl && ( +
+ +
+ + {endpointBaseUrl !== endpointExternalUrl && ( + + )} +
+
+ )} +
+ +
+
+ + - {endpointBaseUrl !== endpointExternalUrl && ( - - )}
- )} -
- - -
-
- - -
-
-
-
- - Python (OpenAI SDK) - - -
-
- - cURL - - -
-
+
+
+ + Python (OpenAI SDK) + + +
+
+ + cURL + + +
+
+ + )} From 597397807951c99a9a63cc84dbfb5aec0801f475 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 9 Apr 2026 02:15:29 +0000 Subject: [PATCH 04/18] fix: code snippet overflow and default to external URL for remote access - Wrap long lines in code snippets (API key in curl was overflowing) - Default snippet base URL to external URL when accessing Studio remotely --- studio/frontend/src/features/chat/chat-page.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 77c76ebb06..12988ea2e2 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -99,7 +99,7 @@ function HighlightedSnippet({ ); return ( -
+
Date: Thu, 9 Apr 2026 09:20:23 +0000 Subject: [PATCH 05/18] fix: Access Endpoint button shows only for GGUF and reflects enabled state - Hide button when loaded model is not a GGUF - Fetch endpoint state on model change so the button stays in sync - Green dot indicator when endpoint is running, red when off --- .../frontend/src/features/chat/chat-page.tsx | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 12988ea2e2..65080bda3b 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -634,8 +634,15 @@ print(completion.choices[0].message.content)`, [], ); + // Keep endpoint state in sync whenever the dialog opens or the loaded + // GGUF model changes, so the status indicator in the header button + // reflects reality without the user having to open the dialog first. useEffect(() => { - if (!endpointDialogOpen) return; + if (!inferenceParams.checkpoint || !activeGgufVariant) { + setEndpointEnabled(false); + setEndpointApiKey(""); + return; + } let cancelled = false; authFetch("/api/inference/access-endpoint") .then((res) => (res.ok ? res.json() : null)) @@ -647,7 +654,12 @@ print(completion.choices[0].message.content)`, return () => { cancelled = true; }; - }, [endpointDialogOpen, applyEndpointData]); + }, [ + endpointDialogOpen, + inferenceParams.checkpoint, + activeGgufVariant, + applyEndpointData, + ]); const handleToggleEndpoint = useCallback(async () => { setEndpointLoading(true); @@ -1076,14 +1088,24 @@ print(completion.choices[0].message.content)`, completionTokens={contextUsage.completionTokens} /> ) : null} - {inferenceParams.checkpoint ? ( + {inferenceParams.checkpoint && activeGgufVariant ? ( ) : null} From 43a633e550522c1ddc9142001d65a4e9664b3f23 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 9 Apr 2026 14:56:01 +0000 Subject: [PATCH 06/18] feat: boot llama-server with --parallel 4 by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raise the default parallel slot count so Studio chat and the external API endpoint can run concurrently without a subprocess restart. Enable/disable of the Access Endpoint is now instant — it only mints or clears the API key, since the parallel slots are already in place from the initial load. --- studio/backend/core/inference/llama_cpp.py | 2 +- studio/backend/routes/inference.py | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index a2947f8770..8008f27b66 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -1285,7 +1285,7 @@ class LlamaCppBackend: "-c", str(effective_ctx) if effective_ctx > 0 else "0", "--parallel", - "1", # Single-user studio, saves VRAM + "4", # Match LM Studio default: supports concurrent Studio chat + external API access "--flash-attn", "on", # Force flash attention for speed ] diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 77a7d6fbfa..a5957f414e 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -949,15 +949,13 @@ async def enable_access_endpoint( request: Request, current_subject: str = Depends(get_current_subject), ): - """Enable external API access: restart llama-server with --parallel 2 and generate an API key.""" + """Enable external API access: generate an API key for the running llama-server.""" llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: raise HTTPException(status_code = 400, detail = "No GGUF model loaded") - # Restart with --parallel 2 for concurrent Studio + external access - await asyncio.to_thread(llama_backend.set_parallel, 2) - - # Generate API key + # llama-server is already running with --parallel 4, so concurrent Studio + # chat + external API is supported without a restart. We just mint a key. api_key = f"sk-unsloth-{_secrets.token_urlsafe(32)}" request.app.state.external_api_key = api_key @@ -976,13 +974,8 @@ async def disable_access_endpoint( request: Request, current_subject: str = Depends(get_current_subject), ): - """Disable external API access: restart llama-server with --parallel 1 and clear the API key.""" + """Disable external API access: clear the API key. llama-server keeps running with --parallel 4.""" request.app.state.external_api_key = None - - llama_backend = get_llama_cpp_backend() - if llama_backend.is_loaded: - await asyncio.to_thread(llama_backend.set_parallel, 1) - return {"enabled": False} From b502ec065a93a80f2d3fd74c5ec0f535506949d3 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Thu, 9 Apr 2026 20:11:39 +0000 Subject: [PATCH 07/18] fix: Access Endpoint code snippets not updating when API key rotates Streamdown memoizes and diffs markdown blocks, so when only the inline API key inside a static code fence changed, it held onto the previously highlighted output and the cURL/Python examples kept showing the stale key while the API Key input showed the new one. Giving Streamdown a React key tied to the markdown string forces a fresh mount whenever the snippet content changes. --- studio/frontend/src/features/chat/chat-page.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 65080bda3b..00d407fab9 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -98,9 +98,14 @@ function HighlightedSnippet({ [language, source], ); + // Streamdown memoizes/diffs markdown blocks and can hold onto previously + // highlighted content when only an inline value inside the code fence + // changes (e.g. the API key). Forcing a remount keyed on the source string + // guarantees the displayed snippet always matches the latest props. return (
Date: Thu, 9 Apr 2026 20:24:19 +0000 Subject: [PATCH 08/18] feat: persistent Access Endpoint API key with Regenerate button The API key used to live only in process memory, so restarting Studio, switching models, or toggling Disable/Enable silently invalidated every external client that was using it. Users had to reopen the dialog, copy the new key, and update their scripts each time. Key state is now stored at ~/.unsloth/studio/auth/access_endpoint.json (atomic write, mode 0600). The file survives backend restarts and model load/unload. Disable preserves the key on disk and only clears it from app.state, so Enable brings the same key back. Loading a different GGUF keeps the endpoint running transparently under the new model identifier. Rotation is an explicit user action: a new JWT-protected POST /access-endpoint/regenerate route mints a fresh key and is wired to a "Regenerate" button next to the API Key field in the dialog. Deliberately not dual-auth so a leaked key cannot rotate itself. --- studio/backend/main.py | 21 ++++ studio/backend/routes/inference.py | 100 +++++++++++++++--- studio/backend/utils/paths/storage_roots.py | 4 + .../frontend/src/features/chat/chat-page.tsx | 46 ++++++-- 4 files changed, 151 insertions(+), 20 deletions(-) diff --git a/studio/backend/main.py b/studio/backend/main.py index ad19ee9679..a14e8931ff 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -131,6 +131,27 @@ async def lifespan(app: FastAPI): print("=" * 60 + "\n") else: app.state.bootstrap_password = storage.get_bootstrap_password() + + # Restore access endpoint API key from disk (if one was left enabled). + # Persisted at ~/.unsloth/studio/auth/access_endpoint.json so the key + # survives backend restarts and external clients don't need to re-fetch + # it every time Studio is restarted. + try: + from routes.inference import _read_endpoint_state + + _ep_state = _read_endpoint_state() + if _ep_state and _ep_state["enabled"]: + app.state.external_api_key = _ep_state["api_key"] + else: + app.state.external_api_key = None + except Exception as exc: + import structlog + + structlog.get_logger(__name__).warning( + "Failed to restore access endpoint state: %s", exc + ) + app.state.external_api_key = None + yield # Cleanup _hw_module.DEVICE = None diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index a5957f414e..8968477309 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -303,9 +303,6 @@ async def load_model( inference_config = load_inference_config(config.identifier) - # Clear any previous API key — user must explicitly enable endpoint - fastapi_request.app.state.external_api_key = None - return LoadResponse( status = "loaded", model = config.identifier, @@ -444,9 +441,6 @@ async def load_model( except Exception: pass - # Clear any previous API key — user must explicitly enable endpoint - fastapi_request.app.state.external_api_key = None - return LoadResponse( status = "loaded", model = config.identifier, @@ -539,9 +533,6 @@ async def unload_model( Routes to the correct backend (llama-server for GGUF, Unsloth otherwise). """ try: - # Clear API key for external access - fastapi_request.app.state.external_api_key = None - # Check if the GGUF backend has this model loaded or is loading it llama_backend = get_llama_cpp_backend() if llama_backend.is_active and ( @@ -926,6 +917,41 @@ def _get_loaded_model_name(): return "default" +def _read_endpoint_state() -> dict | None: + """Return {'api_key': str, 'enabled': bool} or None if file is missing/corrupt.""" + from utils.paths.storage_roots import access_endpoint_state_path + + path = access_endpoint_state_path() + if not path.exists(): + return None + try: + data = json.loads(path.read_text()) + if isinstance(data, dict) and isinstance(data.get("api_key"), str): + return { + "api_key": data["api_key"], + "enabled": bool(data.get("enabled", False)), + } + except Exception as exc: + logger.warning("Failed to read access_endpoint.json: %s", exc) + return None + + +def _write_endpoint_state(api_key: str, enabled: bool) -> None: + """Atomically persist the endpoint state with 0600 perms.""" + from utils.paths.storage_roots import access_endpoint_state_path, ensure_dir + + path = access_endpoint_state_path() + ensure_dir(path.parent) + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps({"api_key": api_key, "enabled": enabled})) + os.chmod(tmp, 0o600) + os.replace(tmp, path) # atomic on POSIX + + +def _mint_api_key() -> str: + return f"sk-unsloth-{_secrets.token_urlsafe(32)}" + + @router.get("/access-endpoint") async def get_access_endpoint( request: Request, @@ -949,14 +975,20 @@ async def enable_access_endpoint( request: Request, current_subject: str = Depends(get_current_subject), ): - """Enable external API access: generate an API key for the running llama-server.""" + """Enable external API access. + + Reuses the persisted API key if one already exists on disk; only mints a + new one on the very first enable or after the file has been removed. + llama-server is already running with --parallel 4, so concurrent Studio + chat + external API is supported without a restart. + """ llama_backend = get_llama_cpp_backend() if not llama_backend.is_loaded: raise HTTPException(status_code = 400, detail = "No GGUF model loaded") - # llama-server is already running with --parallel 4, so concurrent Studio - # chat + external API is supported without a restart. We just mint a key. - api_key = f"sk-unsloth-{_secrets.token_urlsafe(32)}" + state = _read_endpoint_state() + api_key = state["api_key"] if state else _mint_api_key() + _write_endpoint_state(api_key, enabled = True) request.app.state.external_api_key = api_key local_url, external_url = _build_endpoint_urls(request) @@ -974,11 +1006,51 @@ async def disable_access_endpoint( request: Request, current_subject: str = Depends(get_current_subject), ): - """Disable external API access: clear the API key. llama-server keeps running with --parallel 4.""" + """Disable external API access. + + Preserves the key on disk (so the next Enable brings the same key back) + but clears it from in-memory app.state, which causes the dual-auth check + to reject the key on subsequent requests. + """ + state = _read_endpoint_state() + if state: + _write_endpoint_state(state["api_key"], enabled = False) request.app.state.external_api_key = None return {"enabled": False} +@router.post("/access-endpoint/regenerate") +async def regenerate_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Mint a fresh API key, invalidating the previous one. + + JWT-only (deliberately not dual-auth) so a leaked external API key + cannot rotate itself. Only legal while the endpoint is currently enabled + and a GGUF model is loaded. + """ + if getattr(request.app.state, "external_api_key", None) is None: + raise HTTPException(status_code = 400, detail = "Endpoint is not enabled") + + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException(status_code = 400, detail = "No GGUF model loaded") + + api_key = _mint_api_key() + _write_endpoint_state(api_key, enabled = True) + request.app.state.external_api_key = api_key + + local_url, external_url = _build_endpoint_urls(request) + return { + "enabled": True, + "api_key": api_key, + "local_url": local_url, + "external_url": external_url, + "model": _get_loaded_model_name(), + } + + @router.post("/chat/completions") async def openai_chat_completions( payload: ChatCompletionRequest, diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index 4841c5d0a3..9148f3105f 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -50,6 +50,10 @@ def auth_db_path() -> Path: return auth_root() / "auth.db" +def access_endpoint_state_path() -> Path: + return auth_root() / "access_endpoint.json" + + def studio_db_path() -> Path: return studio_root() / "studio.db" diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 00d407fab9..a7a173c22f 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -692,6 +692,28 @@ print(completion.choices[0].message.content)`, } }, [endpointEnabled, applyEndpointData]); + const handleRegenerateKey = useCallback(async () => { + setEndpointLoading(true); + try { + const res = await authFetch( + "/api/inference/access-endpoint/regenerate", + { method: "POST" }, + ); + if (!res.ok) { + const err = await res.json().catch(() => null); + toast.error(err?.detail ?? "Failed to regenerate API key"); + return; + } + const data = await res.json(); + applyEndpointData(data); + toast.success("API key regenerated"); + } catch { + toast.error("Failed to regenerate API key"); + } finally { + setEndpointLoading(false); + } + }, [applyEndpointData]); + const handleCheckpointChange = useCallback( ( value: string, @@ -1247,12 +1269,24 @@ print(completion.choices[0].message.content)`, - +
+ + +
)} From 783ea77b9fdd5f000764aa725502c7ccd7ab3383 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Fri, 10 Apr 2026 10:21:24 +0000 Subject: [PATCH 18/18] fix: use base_url property instead of _base_url in /messages proxy --- studio/backend/routes/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index f503efc0d1..eeacfc91ed 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -2264,7 +2264,7 @@ async def anthropic_messages( detail = "No GGUF model loaded. The Messages API requires a GGUF model.", ) - base_url = llama_backend._base_url + base_url = llama_backend.base_url if not base_url: raise HTTPException(status_code = 503, detail = "llama-server is not running")