feat: explicit enable/disable toggle for API endpoint with --parallel restart
- Add set_parallel(n) to LlamaCppBackend: kills and relaunches llama-server with updated --parallel flag (stores launch cmd/env for replay) - Add POST /access-endpoint/enable: restarts with --parallel 2, generates key - Add POST /access-endpoint/disable: restarts with --parallel 1, clears key - GET /access-endpoint now returns enabled state - Model load clears previous API key (user must explicitly enable) - Frontend dialog shows enable/disable toggle with loading state - Connection details (URLs, key, snippets) only shown when enabled
This commit is contained in:
Roland Tannous
parent
659b08866e
commit
ca1a801300
3 changed files with 282 additions and 108 deletions
|
|
@ -129,6 +129,8 @@ class LlamaCppBackend:
|
|||
self._stdout_thread: Optional[threading.Thread] = None
|
||||
self._cancel_event = threading.Event()
|
||||
self._api_key: Optional[str] = None
|
||||
self._launch_cmd: Optional[list[str]] = None
|
||||
self._launch_env: Optional[dict] = None
|
||||
|
||||
self._kill_orphaned_servers()
|
||||
atexit.register(self._cleanup)
|
||||
|
|
@ -1516,6 +1518,8 @@ class LlamaCppBackend:
|
|||
if gpu_indices is not None:
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_indices)
|
||||
|
||||
self._launch_cmd = list(cmd)
|
||||
self._launch_env = dict(env)
|
||||
self._stdout_lines = []
|
||||
self._process = subprocess.Popen(
|
||||
cmd,
|
||||
|
|
@ -1631,6 +1635,58 @@ class LlamaCppBackend:
|
|||
torch.cuda.empty_cache()
|
||||
return True
|
||||
|
||||
def set_parallel(self, n: int) -> bool:
|
||||
"""Restart llama-server with a different --parallel value.
|
||||
|
||||
Kills the running process and relaunches with the same command
|
||||
but ``--parallel`` swapped. Returns True if the restarted server
|
||||
passes the health check.
|
||||
"""
|
||||
if not self._launch_cmd or not self._launch_env:
|
||||
raise RuntimeError("No launch command stored — cannot restart")
|
||||
|
||||
cmd = list(self._launch_cmd)
|
||||
# Swap --parallel value
|
||||
try:
|
||||
idx = cmd.index("--parallel")
|
||||
cmd[idx + 1] = str(n)
|
||||
except (ValueError, IndexError):
|
||||
cmd.extend(["--parallel", str(n)])
|
||||
|
||||
with self._lock:
|
||||
self._kill_process()
|
||||
self._port = self._find_free_port()
|
||||
# Update port in cmd
|
||||
try:
|
||||
pi = cmd.index("--port")
|
||||
cmd[pi + 1] = str(self._port)
|
||||
except (ValueError, IndexError):
|
||||
cmd.extend(["--port", str(self._port)])
|
||||
|
||||
self._launch_cmd = list(cmd)
|
||||
self._stdout_lines = []
|
||||
self._process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout = subprocess.PIPE,
|
||||
stderr = subprocess.STDOUT,
|
||||
text = True,
|
||||
env = self._launch_env,
|
||||
)
|
||||
self._stdout_thread = threading.Thread(
|
||||
target = self._drain_stdout, daemon = True, name = "llama-stdout"
|
||||
)
|
||||
self._stdout_thread.start()
|
||||
|
||||
if not self._wait_for_health(timeout = 120.0):
|
||||
self._kill_process()
|
||||
raise RuntimeError("llama-server failed to restart with new --parallel")
|
||||
|
||||
self._healthy = True
|
||||
logger.info(
|
||||
f"llama-server restarted with --parallel {n} on port {self._port}"
|
||||
)
|
||||
return True
|
||||
|
||||
def _kill_process(self):
|
||||
"""Terminate the subprocess if running."""
|
||||
if self._process is None:
|
||||
|
|
|
|||
|
|
@ -303,10 +303,8 @@ async def load_model(
|
|||
|
||||
inference_config = load_inference_config(config.identifier)
|
||||
|
||||
# Auto-generate API key for external OpenAI-compatible access
|
||||
fastapi_request.app.state.external_api_key = (
|
||||
f"sk-unsloth-{_secrets.token_urlsafe(32)}"
|
||||
)
|
||||
# Clear any previous API key — user must explicitly enable endpoint
|
||||
fastapi_request.app.state.external_api_key = None
|
||||
|
||||
return LoadResponse(
|
||||
status = "loaded",
|
||||
|
|
@ -446,10 +444,8 @@ async def load_model(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
# Auto-generate API key for external OpenAI-compatible access
|
||||
fastapi_request.app.state.external_api_key = (
|
||||
f"sk-unsloth-{_secrets.token_urlsafe(32)}"
|
||||
)
|
||||
# Clear any previous API key — user must explicitly enable endpoint
|
||||
fastapi_request.app.state.external_api_key = None
|
||||
|
||||
return LoadResponse(
|
||||
status = "loaded",
|
||||
|
|
@ -902,20 +898,11 @@ def _extract_content_parts(
|
|||
# ── Access Endpoint (external OpenAI-compatible access) ──────────
|
||||
|
||||
|
||||
@router.get("/access-endpoint")
|
||||
async def get_access_endpoint(
|
||||
request: Request,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Return API key, local/external URLs, and model info for the Access Endpoint dialog."""
|
||||
api_key = getattr(request.app.state, "external_api_key", None)
|
||||
if not api_key:
|
||||
raise HTTPException(status_code = 400, detail = "No model loaded")
|
||||
|
||||
def _build_endpoint_urls(request: Request):
|
||||
"""Return (local_url, external_url) for the Access Endpoint dialog."""
|
||||
port = getattr(request.app.state, "server_port", None)
|
||||
local_url = f"http://127.0.0.1:{port}/v1" if port else f"{request.base_url}v1"
|
||||
|
||||
# Resolve external IP when bound to all interfaces (same as startup banner)
|
||||
external_url = None
|
||||
bind_host = getattr(request.app.state, "bind_host", None)
|
||||
if bind_host in ("0.0.0.0", "::") and port:
|
||||
|
|
@ -925,22 +912,80 @@ async def get_access_endpoint(
|
|||
if ext_ip and ext_ip not in ("127.0.0.1", "0.0.0.0", "localhost"):
|
||||
external_url = f"http://{ext_ip}:{port}/v1"
|
||||
|
||||
return local_url, external_url
|
||||
|
||||
|
||||
def _get_loaded_model_name():
|
||||
"""Return the currently loaded model identifier."""
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
if llama_backend.is_loaded:
|
||||
return llama_backend.model_identifier
|
||||
backend = get_inference_backend()
|
||||
model = (
|
||||
llama_backend.model_identifier
|
||||
if llama_backend.is_loaded
|
||||
else (list(backend.models.keys())[0] if backend.models else "default")
|
||||
)
|
||||
if backend.models:
|
||||
return list(backend.models.keys())[0]
|
||||
return "default"
|
||||
|
||||
|
||||
@router.get("/access-endpoint")
|
||||
async def get_access_endpoint(
|
||||
request: Request,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Return endpoint state, API key, URLs, and model info."""
|
||||
api_key = getattr(request.app.state, "external_api_key", None)
|
||||
local_url, external_url = _build_endpoint_urls(request)
|
||||
|
||||
return {
|
||||
"enabled": api_key is not None,
|
||||
"api_key": api_key,
|
||||
"local_url": local_url,
|
||||
"external_url": external_url,
|
||||
"model": model,
|
||||
"model": _get_loaded_model_name(),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/access-endpoint/enable")
|
||||
async def enable_access_endpoint(
|
||||
request: Request,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Enable external API access: restart llama-server with --parallel 2 and generate an API key."""
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
if not llama_backend.is_loaded:
|
||||
raise HTTPException(status_code = 400, detail = "No GGUF model loaded")
|
||||
|
||||
# Restart with --parallel 2 for concurrent Studio + external access
|
||||
await asyncio.to_thread(llama_backend.set_parallel, 2)
|
||||
|
||||
# Generate API key
|
||||
api_key = f"sk-unsloth-{_secrets.token_urlsafe(32)}"
|
||||
request.app.state.external_api_key = api_key
|
||||
|
||||
local_url, external_url = _build_endpoint_urls(request)
|
||||
return {
|
||||
"enabled": True,
|
||||
"api_key": api_key,
|
||||
"local_url": local_url,
|
||||
"external_url": external_url,
|
||||
"model": _get_loaded_model_name(),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/access-endpoint/disable")
|
||||
async def disable_access_endpoint(
|
||||
request: Request,
|
||||
current_subject: str = Depends(get_current_subject),
|
||||
):
|
||||
"""Disable external API access: restart llama-server with --parallel 1 and clear the API key."""
|
||||
request.app.state.external_api_key = None
|
||||
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
if llama_backend.is_loaded:
|
||||
await asyncio.to_thread(llama_backend.set_parallel, 1)
|
||||
|
||||
return {"enabled": False}
|
||||
|
||||
|
||||
@router.post("/chat/completions")
|
||||
async def openai_chat_completions(
|
||||
payload: ChatCompletionRequest,
|
||||
|
|
|
|||
|
|
@ -537,6 +537,8 @@ export function ChatPage(): ReactElement {
|
|||
const [view, setView] = useState<ChatView>(getInitialSingleChatView);
|
||||
const [settingsOpen, setSettingsOpen] = useState(false);
|
||||
const [endpointDialogOpen, setEndpointDialogOpen] = useState(false);
|
||||
const [endpointEnabled, setEndpointEnabled] = useState(false);
|
||||
const [endpointLoading, setEndpointLoading] = useState(false);
|
||||
const [endpointLocalUrl, setEndpointLocalUrl] = useState("");
|
||||
const [endpointExternalUrl, setEndpointExternalUrl] = useState<string | null>(null);
|
||||
const [endpointBaseUrl, setEndpointBaseUrl] = useState("");
|
||||
|
|
@ -609,6 +611,23 @@ print(completion.choices[0].message.content)`,
|
|||
[endpointApiKey, endpointBaseUrl, modelAlias],
|
||||
);
|
||||
|
||||
const applyEndpointData = useCallback(
|
||||
(data: {
|
||||
enabled: boolean;
|
||||
api_key: string | null;
|
||||
local_url: string;
|
||||
external_url: string | null;
|
||||
model: string;
|
||||
}) => {
|
||||
setEndpointEnabled(data.enabled);
|
||||
setEndpointApiKey(data.api_key ?? "");
|
||||
setEndpointLocalUrl(data.local_url);
|
||||
setEndpointExternalUrl(data.external_url ?? null);
|
||||
setEndpointBaseUrl(data.local_url);
|
||||
},
|
||||
[],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
if (!endpointDialogOpen) return;
|
||||
let cancelled = false;
|
||||
|
|
@ -616,16 +635,39 @@ print(completion.choices[0].message.content)`,
|
|||
.then((res) => (res.ok ? res.json() : null))
|
||||
.then((data) => {
|
||||
if (cancelled || !data) return;
|
||||
setEndpointApiKey(data.api_key);
|
||||
setEndpointLocalUrl(data.local_url);
|
||||
setEndpointExternalUrl(data.external_url ?? null);
|
||||
setEndpointBaseUrl(data.local_url);
|
||||
applyEndpointData(data);
|
||||
})
|
||||
.catch(() => {});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [endpointDialogOpen]);
|
||||
}, [endpointDialogOpen, applyEndpointData]);
|
||||
|
||||
const handleToggleEndpoint = useCallback(async () => {
|
||||
setEndpointLoading(true);
|
||||
try {
|
||||
const action = endpointEnabled ? "disable" : "enable";
|
||||
const res = await authFetch(`/api/inference/access-endpoint/${action}`, {
|
||||
method: "POST",
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => null);
|
||||
toast.error(err?.detail ?? "Failed to toggle endpoint");
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
applyEndpointData(data);
|
||||
toast.success(
|
||||
data.enabled
|
||||
? "API endpoint enabled"
|
||||
: "API endpoint disabled",
|
||||
);
|
||||
} catch {
|
||||
toast.error("Failed to toggle endpoint");
|
||||
} finally {
|
||||
setEndpointLoading(false);
|
||||
}
|
||||
}, [endpointEnabled, applyEndpointData]);
|
||||
|
||||
const handleCheckpointChange = useCallback(
|
||||
(
|
||||
|
|
@ -1092,93 +1134,124 @@ print(completion.choices[0].message.content)`,
|
|||
<DialogHeader>
|
||||
<DialogTitle>Access Endpoint</DialogTitle>
|
||||
<DialogDescription>
|
||||
Use these OpenAI-compatible settings to connect to the active
|
||||
model from your own code.
|
||||
Serve the active model as an OpenAI-compatible API endpoint
|
||||
for use from your own code, scripts, or other applications.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
<div className="space-y-3">
|
||||
<div className="grid gap-1.5">
|
||||
<label className="text-xs font-medium">Base URL (Local)</label>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
value={endpointLocalUrl}
|
||||
readOnly
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
{endpointBaseUrl !== endpointLocalUrl && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="h-8 shrink-0 text-xs"
|
||||
onClick={() => setEndpointBaseUrl(endpointLocalUrl)}
|
||||
>
|
||||
Use
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex items-center justify-between rounded-md border p-3">
|
||||
<div className="text-sm">
|
||||
{endpointEnabled ? (
|
||||
<span className="font-medium text-green-600 dark:text-green-400">
|
||||
Endpoint is running
|
||||
</span>
|
||||
) : (
|
||||
<span className="text-muted-foreground">
|
||||
Endpoint is off
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{endpointExternalUrl && (
|
||||
<div className="grid gap-1.5">
|
||||
<label className="text-xs font-medium">Base URL (Network)</label>
|
||||
<div className="flex items-center gap-2">
|
||||
<Button
|
||||
variant={endpointEnabled ? "outline" : "default"}
|
||||
size="sm"
|
||||
disabled={endpointLoading}
|
||||
onClick={handleToggleEndpoint}
|
||||
>
|
||||
{endpointLoading
|
||||
? endpointEnabled
|
||||
? "Stopping..."
|
||||
: "Starting..."
|
||||
: endpointEnabled
|
||||
? "Disable"
|
||||
: "Enable"}
|
||||
</Button>
|
||||
</div>
|
||||
{endpointEnabled && (
|
||||
<>
|
||||
<div className="space-y-3">
|
||||
<div className="grid gap-1.5">
|
||||
<label className="text-xs font-medium">Base URL (Local)</label>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
value={endpointLocalUrl}
|
||||
readOnly
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
{endpointBaseUrl !== endpointLocalUrl && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="h-8 shrink-0 text-xs"
|
||||
onClick={() => setEndpointBaseUrl(endpointLocalUrl)}
|
||||
>
|
||||
Use
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{endpointExternalUrl && (
|
||||
<div className="grid gap-1.5">
|
||||
<label className="text-xs font-medium">Base URL (Network)</label>
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
value={endpointExternalUrl}
|
||||
readOnly
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
{endpointBaseUrl !== endpointExternalUrl && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="h-8 shrink-0 text-xs"
|
||||
onClick={() => setEndpointBaseUrl(endpointExternalUrl)}
|
||||
>
|
||||
Use
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="grid gap-1.5">
|
||||
<label htmlFor="endpoint-api-key" className="text-xs font-medium">
|
||||
API Key
|
||||
</label>
|
||||
<Input
|
||||
value={endpointExternalUrl}
|
||||
id="endpoint-api-key"
|
||||
value={endpointApiKey}
|
||||
readOnly
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</div>
|
||||
<div className="grid gap-1.5">
|
||||
<label htmlFor="endpoint-model-alias" className="text-xs font-medium">
|
||||
Model
|
||||
</label>
|
||||
<Input
|
||||
id="endpoint-model-alias"
|
||||
value={modelAlias}
|
||||
readOnly
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
{endpointBaseUrl !== endpointExternalUrl && (
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
className="h-8 shrink-0 text-xs"
|
||||
onClick={() => setEndpointBaseUrl(endpointExternalUrl)}
|
||||
>
|
||||
Use
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="grid gap-1.5">
|
||||
<label htmlFor="endpoint-api-key" className="text-xs font-medium">
|
||||
API Key
|
||||
</label>
|
||||
<Input
|
||||
id="endpoint-api-key"
|
||||
value={endpointApiKey}
|
||||
readOnly
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</div>
|
||||
<div className="grid gap-1.5">
|
||||
<label htmlFor="endpoint-model-alias" className="text-xs font-medium">
|
||||
Model
|
||||
</label>
|
||||
<Input
|
||||
id="endpoint-model-alias"
|
||||
value={modelAlias}
|
||||
readOnly
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<details className="rounded-md border p-2">
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
Python (OpenAI SDK)
|
||||
</summary>
|
||||
<HighlightedSnippet
|
||||
language="python"
|
||||
source={endpointPythonSnippet}
|
||||
/>
|
||||
</details>
|
||||
<details className="rounded-md border p-2">
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
cURL
|
||||
</summary>
|
||||
<HighlightedSnippet language="bash" source={endpointCurlSnippet} />
|
||||
</details>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<details className="rounded-md border p-2">
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
Python (OpenAI SDK)
|
||||
</summary>
|
||||
<HighlightedSnippet
|
||||
language="python"
|
||||
source={endpointPythonSnippet}
|
||||
/>
|
||||
</details>
|
||||
<details className="rounded-md border p-2">
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
cURL
|
||||
</summary>
|
||||
<HighlightedSnippet language="bash" source={endpointCurlSnippet} />
|
||||
</details>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</SidebarProvider>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue