diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index b39f915764..1a5f35e01a 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -5,7 +5,7 @@ import secrets from datetime import datetime, timedelta, timezone from typing import Optional, Tuple -from fastapi import Depends, HTTPException, status +from fastapi import Depends, HTTPException, Request, status from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer import jwt @@ -170,3 +170,61 @@ async def _get_current_subject( status_code = status.HTTP_401_UNAUTHORIZED, detail = "Invalid or expired token", ) + + +# ── Dual-auth for OpenAI-compatible endpoints ─────────────────── + +_optional_security = HTTPBearer(auto_error = False) + + +async def get_current_subject_or_api_key( + request: Request, + credentials: Optional[HTTPAuthorizationCredentials] = Depends(_optional_security), +) -> str: + """Accept a valid JWT or the auto-generated API key for OpenAI-compatible endpoints.""" + if credentials is None: + raise HTTPException( + status_code = status.HTTP_401_UNAUTHORIZED, + detail = "Missing Authorization header", + ) + + token = credentials.credentials + + # Check auto-generated API key (fast path for external consumers) + external_key = getattr(request.app.state, "external_api_key", None) + if external_key and token == external_key: + return "__api_user__" + + # Fall back to JWT validation (Studio frontend sessions) + return await _get_current_subject(credentials, allow_password_change = False) + + +async def get_current_subject_or_api_key_anthropic( + request: Request, + credentials: Optional[HTTPAuthorizationCredentials] = Depends(_optional_security), +) -> str: + """Accept x-api-key header, Authorization: Bearer, or JWT. + + The Anthropic SDK sends ``x-api-key: `` instead of + ``Authorization: Bearer ``. This dependency checks both so that + the ``/v1/messages`` endpoint works with both Anthropic and OpenAI SDKs. + """ + external_key = getattr(request.app.state, "external_api_key", None) + + # Check x-api-key header first (Anthropic SDK default) + x_api_key = request.headers.get("x-api-key") + if x_api_key and external_key and x_api_key == external_key: + return "__api_user__" + + # Fall through to standard Bearer / JWT check + if credentials is None: + raise HTTPException( + status_code = status.HTTP_401_UNAUTHORIZED, + detail = "Missing Authorization or x-api-key header", + ) + + token = credentials.credentials + if external_key and token == external_key: + return "__api_user__" + + return await _get_current_subject(credentials, allow_password_change = False) diff --git a/studio/backend/core/inference/key_exchange.py b/studio/backend/core/inference/key_exchange.py new file mode 100644 index 0000000000..3521ba4351 --- /dev/null +++ b/studio/backend/core/inference/key_exchange.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +RSA + AES-GCM hybrid encryption for transmitting the Access Endpoint +API key to the Studio frontend over plain HTTP. + +Flow (server -> client): + 1. Server generates an RSA-2048 key pair at startup (in memory). + 2. Frontend fetches the public key via GET /access-endpoint/public-key. + 3. Frontend generates a fresh AES-256 session key per reveal. + 4. Frontend RSA-OAEP-encrypts the session key with the server's public key + and POSTs it to /access-endpoint/reveal. + 5. Server decrypts the session key with its private key, then AES-GCM + encrypts the API key payload and returns {iv, ciphertext}. + 6. Frontend AES-GCM-decrypts the payload locally using the session key it + generated. + +The key pair is regenerated on every restart — there is no long-lived secret +material on disk. The only plaintext bearer token ever hitting the wire in +either direction is protected by RSA-OAEP-SHA256 + AES-256-GCM. +""" + +import base64 +import logging +import os + +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding, rsa +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +logger = logging.getLogger(__name__) + +_private_key: rsa.RSAPrivateKey | None = None +_public_key_pem: str | None = None + + +def init_key_pair() -> None: + """Generate an RSA-2048 key pair. Called once at server startup.""" + global _private_key, _public_key_pem + _private_key = rsa.generate_private_key( + public_exponent = 65537, + key_size = 2048, + ) + _public_key_pem = ( + _private_key.public_key() + .public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode("utf-8") + ) + logger.info("RSA key pair generated for access endpoint reveal") + + +def get_public_key_pem() -> str: + """Return the PEM-encoded public key for the frontend.""" + if _public_key_pem is None: + raise RuntimeError("Key pair not initialized. Call init_key_pair() first.") + return _public_key_pem + + +def _decrypt_session_key(encrypted_session_key_b64: str) -> bytes: + """Decrypt a client-supplied AES session key with the server private key.""" + if _private_key is None: + raise RuntimeError("Key pair not initialized. Call init_key_pair() first.") + ciphertext = base64.b64decode(encrypted_session_key_b64) + session_key = _private_key.decrypt( + ciphertext, + padding.OAEP( + mgf = padding.MGF1(algorithm = hashes.SHA256()), + algorithm = hashes.SHA256(), + label = None, + ), + ) + if len(session_key) not in (16, 24, 32): + raise ValueError("Session key must be 128/192/256 bits") + return session_key + + +def encrypt_payload(encrypted_session_key_b64: str, plaintext: str) -> dict[str, str]: + """ + Decrypt the client's session key, then AES-GCM encrypt ``plaintext`` with it. + + Returns a dict ``{"iv": , "ciphertext": }`` where ``ciphertext`` + includes the 16-byte GCM tag appended by the AESGCM primitive. + """ + session_key = _decrypt_session_key(encrypted_session_key_b64) + iv = os.urandom(12) # 96-bit IV is the GCM standard + aesgcm = AESGCM(session_key) + ciphertext = aesgcm.encrypt(iv, plaintext.encode("utf-8"), associated_data = None) + return { + "iv": base64.b64encode(iv).decode("ascii"), + "ciphertext": base64.b64encode(ciphertext).decode("ascii"), + } diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c84ac640df..8008f27b66 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -129,6 +129,8 @@ class LlamaCppBackend: self._stdout_thread: Optional[threading.Thread] = None self._cancel_event = threading.Event() self._api_key: Optional[str] = None + self._launch_cmd: Optional[list[str]] = None + self._launch_env: Optional[dict] = None self._kill_orphaned_servers() atexit.register(self._cleanup) @@ -1283,7 +1285,7 @@ class LlamaCppBackend: "-c", str(effective_ctx) if effective_ctx > 0 else "0", "--parallel", - "1", # Single-user studio, saves VRAM + "4", # Match LM Studio default: supports concurrent Studio chat + external API access "--flash-attn", "on", # Force flash attention for speed ] @@ -1516,6 +1518,8 @@ class LlamaCppBackend: if gpu_indices is not None: env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_indices) + self._launch_cmd = list(cmd) + self._launch_env = dict(env) self._stdout_lines = [] self._process = subprocess.Popen( cmd, @@ -1631,6 +1635,58 @@ class LlamaCppBackend: torch.cuda.empty_cache() return True + def set_parallel(self, n: int) -> bool: + """Restart llama-server with a different --parallel value. + + Kills the running process and relaunches with the same command + but ``--parallel`` swapped. Returns True if the restarted server + passes the health check. + """ + if not self._launch_cmd or not self._launch_env: + raise RuntimeError("No launch command stored — cannot restart") + + cmd = list(self._launch_cmd) + # Swap --parallel value + try: + idx = cmd.index("--parallel") + cmd[idx + 1] = str(n) + except (ValueError, IndexError): + cmd.extend(["--parallel", str(n)]) + + with self._lock: + self._kill_process() + self._port = self._find_free_port() + # Update port in cmd + try: + pi = cmd.index("--port") + cmd[pi + 1] = str(self._port) + except (ValueError, IndexError): + cmd.extend(["--port", str(self._port)]) + + self._launch_cmd = list(cmd) + self._stdout_lines = [] + self._process = subprocess.Popen( + cmd, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + text = True, + env = self._launch_env, + ) + self._stdout_thread = threading.Thread( + target = self._drain_stdout, daemon = True, name = "llama-stdout" + ) + self._stdout_thread.start() + + if not self._wait_for_health(timeout = 120.0): + self._kill_process() + raise RuntimeError("llama-server failed to restart with new --parallel") + + self._healthy = True + logger.info( + f"llama-server restarted with --parallel {n} on port {self._port}" + ) + return True + def _kill_process(self): """Terminate the subprocess if running.""" if self._process is None: diff --git a/studio/backend/main.py b/studio/backend/main.py index ad19ee9679..93df33f172 100644 --- a/studio/backend/main.py +++ b/studio/backend/main.py @@ -93,6 +93,12 @@ async def lifespan(app: FastAPI): # Detect hardware first — sets DEVICE global used everywhere detect_hardware() + # Generate RSA key pair for encrypted access endpoint reveal. + # Lives in memory only — regenerated on each restart. + from core.inference.key_exchange import init_key_pair + + init_key_pair() + from storage.studio_db import cleanup_orphaned_runs try: @@ -131,6 +137,27 @@ async def lifespan(app: FastAPI): print("=" * 60 + "\n") else: app.state.bootstrap_password = storage.get_bootstrap_password() + + # Restore access endpoint API key from disk (if one was left enabled). + # Persisted at ~/.unsloth/studio/auth/access_endpoint.json so the key + # survives backend restarts and external clients don't need to re-fetch + # it every time Studio is restarted. + try: + from routes.inference import _read_endpoint_state + + _ep_state = _read_endpoint_state() + if _ep_state and _ep_state["enabled"]: + app.state.external_api_key = _ep_state["api_key"] + else: + app.state.external_api_key = None + except Exception as exc: + import structlog + + structlog.get_logger(__name__).warning( + "Failed to restore access endpoint state: %s", exc + ) + app.state.external_api_key = None + yield # Cleanup _hw_module.DEVICE = None diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index cf08ecbc12..3b8902fc6d 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -315,7 +315,7 @@ class ChatCompletionRequest(BaseModel): description = "Model identifier (informational; the active model is used)", ) messages: list[ChatMessage] = Field(..., description = "Conversation messages") - stream: bool = Field(True, description = "Whether to stream the response via SSE") + stream: bool = Field(False, description = "Whether to stream the response via SSE") temperature: float = Field(0.6, ge = 0.0, le = 2.0) top_p: float = Field(0.95, ge = 0.0, le = 1.0) max_tokens: Optional[int] = Field( @@ -444,3 +444,71 @@ class ChatCompletion(BaseModel): model: str = "default" choices: list[CompletionChoice] usage: CompletionUsage = Field(default_factory = CompletionUsage) + + +# ── OpenAI Responses API (/v1/responses) ────────────────────── + + +class ResponsesContentPart(BaseModel): + """A content part in the Responses API input/output. + + Supports: + - ``{"type": "input_text", "text": "..."}`` — text input + - ``{"type": "input_image", "image_url": "data:image/...;base64,..."}`` — image input + - ``{"type": "output_text", "text": "..."}`` — text output + """ + + type: str + text: Optional[str] = None + image_url: Optional[str] = None + + +class ResponsesInputMessage(BaseModel): + """A single message in the Responses API ``input`` array.""" + + role: str = "user" + content: Union[str, list[ResponsesContentPart]] + type: str = "message" + + +class ResponsesRequest(BaseModel): + """OpenAI Responses API request.""" + + model: str = Field("default", description = "Model identifier (informational)") + input: Union[str, list[ResponsesInputMessage]] = Field( + ..., description = "Plain text or array of message objects" + ) + stream: bool = Field(False, description = "Whether to stream the response via SSE") + temperature: float = Field(0.6, ge = 0.0, le = 2.0) + top_p: float = Field(0.95, ge = 0.0, le = 1.0) + max_output_tokens: Optional[int] = Field( + None, ge = 1, description = "Maximum tokens to generate" + ) + + +class ResponsesOutputContent(BaseModel): + type: Literal["output_text"] = "output_text" + text: str + + +class ResponsesOutputMessage(BaseModel): + type: Literal["message"] = "message" + role: Literal["assistant"] = "assistant" + content: list[ResponsesOutputContent] + + +class ResponsesUsage(BaseModel): + input_tokens: int = 0 + output_tokens: int = 0 + total_tokens: int = 0 + + +class ResponsesResponse(BaseModel): + """Non-streaming Responses API response.""" + + id: str = Field(default_factory = lambda: f"resp_{uuid.uuid4().hex[:12]}") + object: Literal["response"] = "response" + created_at: int = Field(default_factory = lambda: int(time.time())) + model: str = "default" + output: list[ResponsesOutputMessage] + usage: ResponsesUsage = Field(default_factory = ResponsesUsage) diff --git a/studio/backend/requirements/studio.txt b/studio/backend/requirements/studio.txt index 186ba82fe0..4100ccef03 100644 --- a/studio/backend/requirements/studio.txt +++ b/studio/backend/requirements/studio.txt @@ -15,3 +15,4 @@ huggingface-hub==0.36.2 structlog>=24.1.0 diceware ddgs +cryptography>=42.0.0 diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 30ff7da49c..eeacfc91ed 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -6,6 +6,7 @@ Inference API routes for model loading and text generation. """ import os +import secrets as _secrets import sys import time import uuid @@ -83,8 +84,15 @@ from models.inference import ( CompletionUsage, ValidateModelRequest, ValidateModelResponse, + ResponsesRequest, + ResponsesResponse, + ResponsesOutputMessage, + ResponsesOutputContent, + ResponsesUsage, + ResponsesInputMessage, + ResponsesContentPart, ) -from auth.authentication import get_current_subject +from auth.authentication import get_current_subject, get_current_subject_or_api_key, get_current_subject_or_api_key_anthropic import io import wave @@ -121,6 +129,7 @@ def get_llama_cpp_backend() -> LlamaCppBackend: @router.post("/load", response_model = LoadResponse) async def load_model( request: LoadRequest, + fastapi_request: Request, current_subject: str = Depends(get_current_subject), ): """ @@ -523,6 +532,7 @@ async def validate_model( @router.post("/unload", response_model = UnloadResponse) async def unload_model( request: UnloadRequest, + fastapi_request: Request, current_subject: str = Depends(get_current_subject), ): """ @@ -883,11 +893,233 @@ def _extract_content_parts( return system_prompt, chat_messages, first_image_b64 +# ── Access Endpoint (external OpenAI-compatible access) ────────── + + +def _build_endpoint_urls(request: Request): + """Return (local_url, external_url) for the Access Endpoint dialog.""" + port = getattr(request.app.state, "server_port", None) + local_url = f"http://127.0.0.1:{port}/v1" if port else f"{request.base_url}v1" + + external_url = None + bind_host = getattr(request.app.state, "bind_host", None) + if bind_host in ("0.0.0.0", "::") and port: + from run import _resolve_external_ip + + ext_ip = _resolve_external_ip() + if ext_ip and ext_ip not in ("127.0.0.1", "0.0.0.0", "localhost"): + external_url = f"http://{ext_ip}:{port}/v1" + + return local_url, external_url + + +def _get_loaded_model_name(): + """Return the currently loaded model identifier.""" + llama_backend = get_llama_cpp_backend() + if llama_backend.is_loaded: + return llama_backend.model_identifier + backend = get_inference_backend() + if backend.models: + return list(backend.models.keys())[0] + return "default" + + +def _read_endpoint_state() -> dict | None: + """Return {'api_key': str, 'enabled': bool} or None if file is missing/corrupt.""" + from utils.paths.storage_roots import access_endpoint_state_path + + path = access_endpoint_state_path() + if not path.exists(): + return None + try: + data = json.loads(path.read_text()) + if isinstance(data, dict) and isinstance(data.get("api_key"), str): + return { + "api_key": data["api_key"], + "enabled": bool(data.get("enabled", False)), + } + except Exception as exc: + logger.warning("Failed to read access_endpoint.json: %s", exc) + return None + + +def _write_endpoint_state(api_key: str, enabled: bool) -> None: + """Atomically persist the endpoint state with 0600 perms.""" + from utils.paths.storage_roots import access_endpoint_state_path, ensure_dir + + path = access_endpoint_state_path() + ensure_dir(path.parent) + tmp = path.with_suffix(".json.tmp") + tmp.write_text(json.dumps({"api_key": api_key, "enabled": enabled})) + os.chmod(tmp, 0o600) + os.replace(tmp, path) # atomic on POSIX + + +def _mint_api_key() -> str: + return f"sk-unsloth-{_secrets.token_urlsafe(32)}" + + +@router.get("/access-endpoint") +async def get_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Return endpoint state, URLs, and model info. + + The API key itself is NOT returned here — the frontend must fetch it via + the encrypted POST /access-endpoint/reveal flow so it never travels as + plaintext over the wire. + """ + api_key = getattr(request.app.state, "external_api_key", None) + local_url, external_url = _build_endpoint_urls(request) + + return { + "enabled": api_key is not None, + "local_url": local_url, + "external_url": external_url, + "model": _get_loaded_model_name(), + } + + +@router.post("/access-endpoint/enable") +async def enable_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Enable external API access. + + Reuses the persisted API key if one already exists on disk; only mints a + new one on the very first enable or after the file has been removed. + llama-server is already running with --parallel 4, so concurrent Studio + chat + external API is supported without a restart. + """ + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException(status_code = 400, detail = "No GGUF model loaded") + + state = _read_endpoint_state() + api_key = state["api_key"] if state else _mint_api_key() + _write_endpoint_state(api_key, enabled = True) + request.app.state.external_api_key = api_key + + local_url, external_url = _build_endpoint_urls(request) + return { + "enabled": True, + "local_url": local_url, + "external_url": external_url, + "model": _get_loaded_model_name(), + } + + +@router.post("/access-endpoint/disable") +async def disable_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Disable external API access. + + Preserves the key on disk (so the next Enable brings the same key back) + but clears it from in-memory app.state, which causes the dual-auth check + to reject the key on subsequent requests. + """ + state = _read_endpoint_state() + if state: + _write_endpoint_state(state["api_key"], enabled = False) + request.app.state.external_api_key = None + return {"enabled": False} + + +@router.post("/access-endpoint/regenerate") +async def regenerate_access_endpoint( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Mint a fresh API key, invalidating the previous one. + + JWT-only (deliberately not dual-auth) so a leaked external API key + cannot rotate itself. Only legal while the endpoint is currently enabled + and a GGUF model is loaded. + """ + if getattr(request.app.state, "external_api_key", None) is None: + raise HTTPException(status_code = 400, detail = "Endpoint is not enabled") + + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException(status_code = 400, detail = "No GGUF model loaded") + + api_key = _mint_api_key() + _write_endpoint_state(api_key, enabled = True) + request.app.state.external_api_key = api_key + + local_url, external_url = _build_endpoint_urls(request) + return { + "enabled": True, + "local_url": local_url, + "external_url": external_url, + "model": _get_loaded_model_name(), + } + + +@router.get("/access-endpoint/public-key") +async def access_endpoint_public_key( + current_subject: str = Depends(get_current_subject), +): + """Return the server's RSA public key (PEM) for the reveal handshake. + + JWT-protected so only authenticated Studio sessions can start the + reveal flow. + """ + from core.inference.key_exchange import get_public_key_pem + + return {"public_key": get_public_key_pem()} + + +@router.post("/access-endpoint/reveal") +async def access_endpoint_reveal( + request: Request, + current_subject: str = Depends(get_current_subject), +): + """Return the live API key, AES-GCM encrypted with a client-supplied session key. + + Request body: + { "encrypted_session_key": "" } + + Response body: + { "iv": "", "ciphertext": "" } + + The plaintext inside the ciphertext is the API key string. If the endpoint + is disabled, returns 404 so the UI can hide the field rather than reveal + an inactive key. + """ + from core.inference.key_exchange import encrypt_payload + + api_key = getattr(request.app.state, "external_api_key", None) + if not api_key: + raise HTTPException(status_code = 404, detail = "Endpoint is not enabled") + + try: + body = await request.json() + except Exception: + raise HTTPException(status_code = 400, detail = "Invalid JSON body") + + encrypted_session_key = body.get("encrypted_session_key") if isinstance(body, dict) else None + if not isinstance(encrypted_session_key, str) or not encrypted_session_key: + raise HTTPException( + status_code = 400, detail = "Missing encrypted_session_key" + ) + + try: + return encrypt_payload(encrypted_session_key, api_key) + except Exception as exc: + logger.warning("Access endpoint reveal failed: %s", exc) + raise HTTPException(status_code = 400, detail = "Failed to decrypt session key") + + @router.post("/chat/completions") async def openai_chat_completions( payload: ChatCompletionRequest, request: Request, - current_subject: str = Depends(get_current_subject), + current_subject: str = Depends(get_current_subject_or_api_key), ): """ OpenAI-compatible chat completions endpoint. @@ -1483,11 +1715,22 @@ async def openai_chat_completions( else: try: full_text = "" + _ns_usage = None for token in gguf_generate(): if isinstance(token, dict): - continue # skip metadata dict in non-streaming path + if token.get("type") == "metadata": + _ns_usage = token.get("usage") + continue full_text = token + usage_obj = None + if _ns_usage: + usage_obj = CompletionUsage( + prompt_tokens = _ns_usage.get("prompt_tokens", 0), + completion_tokens = _ns_usage.get("completion_tokens", 0), + total_tokens = _ns_usage.get("total_tokens", 0), + ) + response = ChatCompletion( id = completion_id, created = created, @@ -1498,6 +1741,7 @@ async def openai_chat_completions( finish_reason = "stop", ) ], + usage = usage_obj, ) return JSONResponse(content = response.model_dump()) @@ -1777,13 +2021,304 @@ async def serve_sandbox_file( # ===================================================================== +# ===================================================================== +# OpenAI Responses API (/responses → /v1/responses) +# ===================================================================== + + +def _responses_input_to_messages( + input_data: str | list[ResponsesInputMessage], +) -> tuple[list[dict], str | None]: + """Convert Responses API ``input`` to OpenAI-style message dicts. + + Returns: + messages: List of ``{"role": ..., "content": ...}`` dicts. + image_b64: Base64 data of the first ``input_image`` found, or ``None``. + """ + if isinstance(input_data, str): + return [{"role": "user", "content": input_data}], None + + messages: list[dict] = [] + first_image_b64: str | None = None + for item in input_data: + if isinstance(item.content, str): + messages.append({"role": item.role, "content": item.content}) + elif isinstance(item.content, list): + text_parts: list[str] = [] + for part in item.content: + if part.type == "input_text" and part.text: + text_parts.append(part.text) + elif part.type == "input_image" and part.image_url and first_image_b64 is None: + url = part.image_url + if url.startswith("data:"): + first_image_b64 = url.split(",", 1)[1] if "," in url else None + else: + logger.warning( + "Remote image URLs not yet supported in /responses: %s...", + url[:80], + ) + combined_text = "\n".join(text_parts) if text_parts else "" + messages.append({"role": item.role, "content": combined_text}) + return messages, first_image_b64 + + +@router.post("/responses") +async def openai_responses( + payload: ResponsesRequest, + request: Request, + current_subject: str = Depends(get_current_subject_or_api_key), +): + """ + OpenAI Responses API endpoint. + + Accepts the Responses API ``input`` format and returns a Response object. + Translates internally to the GGUF chat completion path. Only GGUF models + are supported (same as ``/chat/completions`` for external API access). + """ + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException( + status_code = 400, + detail = "No GGUF model loaded. The Responses API requires a GGUF model.", + ) + + model_name = llama_backend.model_identifier or payload.model + messages, image_b64 = _responses_input_to_messages(payload.input) + cancel_event = threading.Event() + response_id = f"resp_{uuid.uuid4().hex[:12]}" + created_at = int(time.time()) + + def responses_generate(): + return llama_backend.generate_chat_completion( + messages = messages, + image_b64 = image_b64, + temperature = payload.temperature, + top_p = payload.top_p, + top_k = 40, + min_p = 0.0, + max_tokens = payload.max_output_tokens, + repetition_penalty = 1.0, + presence_penalty = 0.0, + cancel_event = cancel_event, + enable_thinking = False, + ) + + if payload.stream: + + async def responses_stream_events(): + try: + # response.created event + created_event = { + "type": "response.created", + "response": { + "id": response_id, + "object": "response", + "created_at": created_at, + "model": model_name, + "output": [], + "usage": None, + }, + } + yield f"event: response.created\ndata: {json.dumps(created_event)}\n\n" + + _sentinel = object() + gen = responses_generate() + prev_text = "" + _usage = None + output_index = 0 + content_index = 0 + + while True: + if await request.is_disconnected(): + cancel_event.set() + return + cumulative = await asyncio.to_thread(next, gen, _sentinel) + if cumulative is _sentinel: + break + if isinstance(cumulative, dict): + if cumulative.get("type") == "metadata": + _usage = cumulative.get("usage") + continue + new_text = cumulative[len(prev_text):] + prev_text = cumulative + if not new_text: + continue + delta_event = { + "type": "response.output_text.delta", + "output_index": output_index, + "content_index": content_index, + "delta": new_text, + } + yield f"event: response.output_text.delta\ndata: {json.dumps(delta_event)}\n\n" + + # response.completed event + usage_data = None + if _usage: + usage_data = { + "input_tokens": _usage.get("prompt_tokens", 0), + "output_tokens": _usage.get("completion_tokens", 0), + "total_tokens": _usage.get("total_tokens", 0), + } + completed_event = { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "created_at": created_at, + "model": model_name, + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": prev_text}], + } + ], + "usage": usage_data, + }, + } + yield f"event: response.completed\ndata: {json.dumps(completed_event)}\n\n" + + except asyncio.CancelledError: + cancel_event.set() + raise + except Exception as e: + logger.error(f"Error during Responses API streaming: {e}", exc_info = True) + error_event = { + "type": "error", + "error": {"message": _friendly_error(e), "type": "server_error"}, + } + yield f"event: error\ndata: {json.dumps(error_event)}\n\n" + + return StreamingResponse( + responses_stream_events(), + media_type = "text/event-stream", + headers = { + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + else: + try: + full_text = "" + _usage = None + for token in responses_generate(): + if isinstance(token, dict): + if token.get("type") == "metadata": + _usage = token.get("usage") + continue + full_text = token + + usage_obj = ResponsesUsage() + if _usage: + usage_obj = ResponsesUsage( + input_tokens = _usage.get("prompt_tokens", 0), + output_tokens = _usage.get("completion_tokens", 0), + total_tokens = _usage.get("total_tokens", 0), + ) + + response = ResponsesResponse( + id = response_id, + created_at = created_at, + model = model_name, + output = [ + ResponsesOutputMessage( + content = [ResponsesOutputContent(text = full_text)] + ) + ], + usage = usage_obj, + ) + return JSONResponse(content = response.model_dump()) + + except Exception as e: + logger.error(f"Error during Responses API completion: {e}", exc_info = True) + raise HTTPException(status_code = 500, detail = _friendly_error(e)) + + +# ===================================================================== +# Anthropic Messages API (/messages → /v1/messages) +# ===================================================================== + + +@router.post("/messages") +async def anthropic_messages( + request: Request, + current_subject: str = Depends(get_current_subject_or_api_key_anthropic), +): + """ + Anthropic-compatible Messages API endpoint. + + Proxies the request directly to llama-server's native ``/v1/messages`` + handler, which supports the full Anthropic Messages format including + streaming (``message_start``, ``content_block_delta``, etc.). + + Accepts auth via ``x-api-key`` header (Anthropic SDK) or + ``Authorization: Bearer`` (OpenAI SDK / general). + """ + import httpx + + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException( + status_code = 400, + detail = "No GGUF model loaded. The Messages API requires a GGUF model.", + ) + + base_url = llama_backend.base_url + if not base_url: + raise HTTPException(status_code = 503, detail = "llama-server is not running") + + body = await request.body() + # Detect streaming from the JSON body + try: + payload = json.loads(body) + is_stream = payload.get("stream", False) + except Exception: + is_stream = False + + target_url = f"{base_url}/v1/messages" + + if is_stream: + + async def proxy_stream(): + async with httpx.AsyncClient() as client: + async with client.stream( + "POST", + target_url, + content = body, + headers = {"Content-Type": "application/json"}, + timeout = 300.0, + ) as resp: + async for chunk in resp.aiter_bytes(): + yield chunk + + return StreamingResponse( + proxy_stream(), + media_type = "text/event-stream", + headers = { + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + else: + async with httpx.AsyncClient() as client: + resp = await client.post( + target_url, + content = body, + headers = {"Content-Type": "application/json"}, + timeout = 300.0, + ) + return JSONResponse(content = resp.json(), status_code = resp.status_code) + + # OpenAI-Compatible Models Listing (/models → /v1/models) # ===================================================================== @router.get("/models") async def openai_list_models( - current_subject: str = Depends(get_current_subject), + current_subject: str = Depends(get_current_subject_or_api_key), ): """ OpenAI-compatible model listing endpoint. diff --git a/studio/backend/run.py b/studio/backend/run.py index 86c1194661..db2b83b300 100644 --- a/studio/backend/run.py +++ b/studio/backend/run.py @@ -331,6 +331,7 @@ def run_server( # binds (port==0) leave it unset and let request handlers fall back # to the ASGI request scope or request.base_url. app.state.server_port = port if port and port > 0 else None + app.state.bind_host = host # Run server in a daemon thread def _run(): diff --git a/studio/backend/utils/paths/storage_roots.py b/studio/backend/utils/paths/storage_roots.py index 4841c5d0a3..9148f3105f 100644 --- a/studio/backend/utils/paths/storage_roots.py +++ b/studio/backend/utils/paths/storage_roots.py @@ -50,6 +50,10 @@ def auth_db_path() -> Path: return auth_root() / "auth.db" +def access_endpoint_state_path() -> Path: + return auth_root() / "access_endpoint.json" + + def studio_db_path() -> Path: return studio_root() / "studio.db" diff --git a/studio/frontend/package.json b/studio/frontend/package.json index d9bab4de7e..3f3bf33228 100644 --- a/studio/frontend/package.json +++ b/studio/frontend/package.json @@ -58,6 +58,7 @@ "motion": "^12.34.0", "next": "^16.1.6", "next-themes": "^0.4.6", + "node-forge": "^1.4.0", "radix-ui": "^1.4.3", "react": "^19.2.4", "react-day-picker": "^9.13.2", @@ -80,6 +81,7 @@ "@eslint/js": "^9.39.1", "@types/js-yaml": "^4.0.9", "@types/node": "^24.10.1", + "@types/node-forge": "^1.3.14", "@types/react": "^19.2.5", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^6.0.1", diff --git a/studio/frontend/src/features/chat/api/access-endpoint-crypto.ts b/studio/frontend/src/features/chat/api/access-endpoint-crypto.ts new file mode 100644 index 0000000000..cbe22193b9 --- /dev/null +++ b/studio/frontend/src/features/chat/api/access-endpoint-crypto.ts @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +/** + * Encrypted reveal flow for the Access Endpoint API key. + * + * Studio is often served over plain HTTP, which rules out WebCrypto + * (`crypto.subtle` is HTTPS-only outside of localhost). We therefore use + * node-forge for both RSA-OAEP (session key wrap) and AES-256-GCM (payload + * decrypt). + * + * Flow: + * 1. Fetch the server's RSA public key (PEM) — cached for the session. + * 2. Generate a random 32-byte AES-256 session key in the browser. + * 3. RSA-OAEP-SHA256 encrypt the session key with the server public key. + * 4. POST the encrypted session key to /api/inference/access-endpoint/reveal. + * 5. Server decrypts the session key, AES-GCM encrypts the API key and + * returns {iv, ciphertext}. + * 6. Decrypt ciphertext locally using the session key we generated. + * + * The session key never leaves the browser in plaintext, and the API key + * never leaves the server in plaintext. + */ + +import forge from "node-forge"; + +import { authFetch } from "@/features/auth"; + +let cachedPublicKeyPem: string | null = null; +let cachedForgeKey: forge.pki.rsa.PublicKey | null = null; + +export function clearAccessEndpointPublicKeyCache(): void { + cachedPublicKeyPem = null; + cachedForgeKey = null; +} + +async function fetchAccessEndpointPublicKey( + forceRefresh = false, +): Promise { + if (!forceRefresh && cachedForgeKey) { + return cachedForgeKey; + } + const response = await authFetch("/api/inference/access-endpoint/public-key"); + if (!response.ok) { + throw new Error(`Failed to fetch public key: ${response.status}`); + } + const body = (await response.json()) as { public_key?: string }; + const publicKeyPem = body.public_key?.trim(); + if (!publicKeyPem) { + throw new Error("Public key missing from response"); + } + if (!forceRefresh && cachedPublicKeyPem === publicKeyPem && cachedForgeKey) { + return cachedForgeKey; + } + const forgeKey = forge.pki.publicKeyFromPem(publicKeyPem); + cachedPublicKeyPem = publicKeyPem; + cachedForgeKey = forgeKey; + return forgeKey; +} + +/** + * Perform the encrypted reveal handshake and return the plaintext API key. + * Returns `null` if the endpoint is disabled (HTTP 404 from reveal). + */ +export async function revealAccessEndpointKey(): Promise { + // Step 1: fetch server public key (cached per session) + let publicKey: forge.pki.rsa.PublicKey; + try { + publicKey = await fetchAccessEndpointPublicKey(); + } catch { + // Refresh cache once on failure (server may have restarted with new keypair) + publicKey = await fetchAccessEndpointPublicKey(true); + } + + // Step 2: generate AES-256 session key + const sessionKeyBytes = forge.random.getBytesSync(32); + + // Step 3: RSA-OAEP-SHA256 wrap the session key + const encryptedSessionKeyBytes = publicKey.encrypt( + sessionKeyBytes, + "RSA-OAEP", + { + md: forge.md.sha256.create(), + mgf1: { md: forge.md.sha256.create() }, + }, + ); + const encryptedSessionKeyB64 = forge.util.encode64(encryptedSessionKeyBytes); + + // Step 4: POST to reveal endpoint + const response = await authFetch( + "/api/inference/access-endpoint/reveal", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ encrypted_session_key: encryptedSessionKeyB64 }), + }, + ); + + if (response.status === 404) { + return null; // Endpoint not enabled + } + if (!response.ok) { + // A restart can invalidate the cached public key — retry once with a + // fresh key before surrendering. + clearAccessEndpointPublicKeyCache(); + throw new Error(`Reveal failed: ${response.status}`); + } + + const body = (await response.json()) as { iv?: string; ciphertext?: string }; + if (!body.iv || !body.ciphertext) { + throw new Error("Reveal response missing iv/ciphertext"); + } + + // Step 5: AES-GCM decrypt + const ivBytes = forge.util.decode64(body.iv); + const ciphertextWithTag = forge.util.decode64(body.ciphertext); + + // forge's GCM API wants ciphertext and tag separated (tag is the last 16 bytes) + const tagLength = 16; + if (ciphertextWithTag.length < tagLength) { + throw new Error("Ciphertext too short for GCM tag"); + } + const ctBytes = ciphertextWithTag.slice( + 0, + ciphertextWithTag.length - tagLength, + ); + const tagBytes = ciphertextWithTag.slice( + ciphertextWithTag.length - tagLength, + ); + + const decipher = forge.cipher.createDecipher("AES-GCM", sessionKeyBytes); + decipher.start({ + iv: ivBytes, + tagLength: tagLength * 8, + tag: forge.util.createBuffer(tagBytes), + }); + decipher.update(forge.util.createBuffer(ctBytes)); + const ok = decipher.finish(); + if (!ok) { + throw new Error("AES-GCM authentication failed"); + } + return decipher.output.toString(); +} diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index cf1ba11d7b..901adc02bb 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -8,6 +8,14 @@ import { } from "@/components/assistant-ui/model-selector"; import { Thread } from "@/components/assistant-ui/thread"; import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; import { Sheet, SheetContent, @@ -33,6 +41,7 @@ import { Settings04Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; +import { code as codePlugin } from "@streamdown/code"; import { type CSSProperties, type ReactElement, @@ -44,7 +53,10 @@ import { useRef, useState, } from "react"; +import { Streamdown } from "streamdown"; import { toast } from "sonner"; +import { authFetch } from "@/features/auth"; +import { revealAccessEndpointKey } from "./api/access-endpoint-crypto"; import { listLocalModels } from "./api/chat-api"; import { ChatSettingsPanel } from "./chat-settings-sheet"; import { ContextUsageBar } from "./components/context-usage-bar"; @@ -73,6 +85,81 @@ type LoraCandidate = { updatedAt?: number; }; +const SHIKI_THEME = ["github-light", "github-dark"] as ["github-light", "github-dark"]; + +function HighlightedSnippet({ + language, + source, +}: { + language: "python" | "bash"; + source: string; +}) { + const [copied, setCopied] = useState(false); + const markdown = useMemo( + () => `\`\`\`${language}\n${source}\n\`\`\``, + [language, source], + ); + + const handleCopy = useCallback( + (e: React.MouseEvent) => { + e.preventDefault(); + e.stopPropagation(); + + // Always use the textarea/execCommand approach — it works on plain HTTP + // inside a synchronous user-gesture handler. navigator.clipboard is + // HTTPS-only on non-localhost origins and silently fails. + const ta = document.createElement("textarea"); + ta.value = source; + // Must be visible (off-screen but not display:none) for execCommand to work + ta.setAttribute("readonly", ""); + ta.style.position = "fixed"; + ta.style.left = "-9999px"; + ta.style.top = "-9999px"; + ta.style.opacity = "0"; + document.body.appendChild(ta); + ta.focus(); + ta.select(); + // setSelectionRange needed for iOS + ta.setSelectionRange(0, ta.value.length); + const ok = document.execCommand("copy"); + document.body.removeChild(ta); + + if (ok) { + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } + }, + [source], + ); + + // Streamdown memoizes/diffs markdown blocks and can hold onto previously + // highlighted content when only an inline value inside the code fence + // changes (e.g. the API key). Forcing a remount keyed on the source string + // guarantees the displayed snippet always matches the latest props. + return ( +
+ +
+ + {markdown} + +
+
+ ); +} + function normalizeModelRef(value: string | null | undefined): string { return value?.trim().toLowerCase() ?? ""; } @@ -497,6 +584,13 @@ export function ChatPage(): ReactElement { // explicitly sets a nonce in handleNewThread. const [view, setView] = useState(getInitialSingleChatView); const [settingsOpen, setSettingsOpen] = useState(false); + const [endpointDialogOpen, setEndpointDialogOpen] = useState(false); + const [endpointEnabled, setEndpointEnabled] = useState(false); + const [endpointLoading, setEndpointLoading] = useState(false); + const [endpointLocalUrl, setEndpointLocalUrl] = useState(""); + const [endpointExternalUrl, setEndpointExternalUrl] = useState(null); + const [endpointBaseUrl, setEndpointBaseUrl] = useState(""); + const [endpointApiKey, setEndpointApiKey] = useState(""); const [modelSelectorOpen, setModelSelectorOpen] = useState(false); const [modelSelectorLocked, setModelSelectorLocked] = useState(false); const [sidebarOpen, setSidebarOpen] = useState(true); @@ -537,6 +631,214 @@ export function ChatPage(): ReactElement { const canCompare = useMemo(() => { return Boolean(inferenceParams.checkpoint); }, [inferenceParams.checkpoint]); + const modelAlias = inferenceParams.checkpoint || "unsloth/your-model-alias"; + const endpointPythonSnippet = useMemo( + () => `from openai import OpenAI + +client = OpenAI( + base_url="${endpointBaseUrl}", + api_key="${endpointApiKey}", +) + +completion = client.chat.completions.create( + model="${modelAlias}", + messages=[{"role": "user", "content": "What is 2+2?"}], +) + +print(completion.choices[0].message.content)`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const endpointCurlSnippet = useMemo( + () => `curl ${endpointBaseUrl}/chat/completions \\ + -H "Content-Type: application/json" \\ + -H "Authorization: Bearer ${endpointApiKey}" \\ + -d '{ + "model": "${modelAlias}", + "messages": [{"role": "user", "content": "What is 2+2?"}] + }'`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const responsesPythonSnippet = useMemo( + () => `from openai import OpenAI + +client = OpenAI( + base_url="${endpointBaseUrl}", + api_key="${endpointApiKey}", +) + +response = client.responses.create( + model="${modelAlias}", + input=[{ + "role": "user", + "content": [{"type": "input_text", "text": "What is 2+2?"}], + }], +) + +print(response.output[0].content[0].text)`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const responsesCurlSnippet = useMemo( + () => `curl ${endpointBaseUrl}/responses \\ + -H "Content-Type: application/json" \\ + -H "Authorization: Bearer ${endpointApiKey}" \\ + -d '{ + "model": "${modelAlias}", + "input": [{ + "role": "user", + "content": [{"type": "input_text", "text": "What is 2+2?"}] + }] + }'`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const anthropicPythonSnippet = useMemo( + () => `from anthropic import Anthropic + +client = Anthropic( + base_url="${endpointBaseUrl}", + api_key="${endpointApiKey}", +) + +message = client.messages.create( + model="${modelAlias}", + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], +) + +print(message.content[0].text)`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const anthropicCurlSnippet = useMemo( + () => `curl ${endpointBaseUrl}/messages \\ + -H "Content-Type: application/json" \\ + -H "x-api-key: ${endpointApiKey}" \\ + -d '{ + "model": "${modelAlias}", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "What is 2+2?"}] + }'`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const [endpointApiTab, setEndpointApiTab] = useState<"completions" | "responses" | "messages">("completions"); + + const applyEndpointData = useCallback( + (data: { + enabled: boolean; + local_url: string; + external_url: string | null; + model: string; + }) => { + setEndpointEnabled(data.enabled); + if (!data.enabled) { + setEndpointApiKey(""); + } + setEndpointLocalUrl(data.local_url); + setEndpointExternalUrl(data.external_url ?? null); + // Default to external URL when accessing Studio remotely + const isRemote = + typeof window !== "undefined" && + !["localhost", "127.0.0.1", "[::1]"].includes(window.location.hostname); + setEndpointBaseUrl( + isRemote && data.external_url ? data.external_url : data.local_url, + ); + }, + [], + ); + + // Fetch the API key via the encrypted reveal handshake. Called after any + // endpoint state change that leaves the endpoint enabled (open dialog, + // enable, regenerate). The plaintext key only lives in React state. + const revealAndSetApiKey = useCallback(async () => { + try { + const key = await revealAccessEndpointKey(); + setEndpointApiKey(key ?? ""); + } catch { + setEndpointApiKey(""); + toast.error("Failed to fetch API key"); + } + }, []); + + // Keep endpoint state in sync whenever the dialog opens or the loaded + // GGUF model changes, so the status indicator in the header button + // reflects reality without the user having to open the dialog first. + useEffect(() => { + if (!inferenceParams.checkpoint || !activeGgufVariant) { + setEndpointEnabled(false); + setEndpointApiKey(""); + return; + } + let cancelled = false; + authFetch("/api/inference/access-endpoint") + .then((res) => (res.ok ? res.json() : null)) + .then(async (data) => { + if (cancelled || !data) return; + applyEndpointData(data); + if (data.enabled) { + const key = await revealAccessEndpointKey().catch(() => null); + if (!cancelled) setEndpointApiKey(key ?? ""); + } + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, [ + endpointDialogOpen, + inferenceParams.checkpoint, + activeGgufVariant, + applyEndpointData, + ]); + + const handleToggleEndpoint = useCallback(async () => { + setEndpointLoading(true); + try { + const action = endpointEnabled ? "disable" : "enable"; + const res = await authFetch(`/api/inference/access-endpoint/${action}`, { + method: "POST", + }); + if (!res.ok) { + const err = await res.json().catch(() => null); + toast.error(err?.detail ?? "Failed to toggle endpoint"); + return; + } + const data = await res.json(); + applyEndpointData(data); + if (data.enabled) { + await revealAndSetApiKey(); + } + toast.success( + data.enabled + ? "API endpoint enabled" + : "API endpoint disabled", + ); + } catch { + toast.error("Failed to toggle endpoint"); + } finally { + setEndpointLoading(false); + } + }, [endpointEnabled, applyEndpointData, revealAndSetApiKey]); + + const handleRegenerateKey = useCallback(async () => { + setEndpointLoading(true); + try { + const res = await authFetch( + "/api/inference/access-endpoint/regenerate", + { method: "POST" }, + ); + if (!res.ok) { + const err = await res.json().catch(() => null); + toast.error(err?.detail ?? "Failed to regenerate API key"); + return; + } + const data = await res.json(); + applyEndpointData(data); + await revealAndSetApiKey(); + toast.success("API key regenerated"); + } catch { + toast.error("Failed to regenerate API key"); + } finally { + setEndpointLoading(false); + } + }, [applyEndpointData, revealAndSetApiKey]); const handleCheckpointChange = useCallback( ( @@ -641,6 +943,13 @@ export function ChatPage(): ReactElement { const openSettings = useCallback(() => setSettingsOpen(true), []); const closeSettings = useCallback(() => setSettingsOpen(false), []); const openSidebar = useCallback(() => setSidebarOpen(true), []); + const handleOpenEndpointDialog = useCallback(() => { + if (!inferenceParams.checkpoint) { + toast.message("Load a model first to access endpoint details."); + return; + } + setEndpointDialogOpen(true); + }, [inferenceParams.checkpoint]); const enterCompare = useCallback(() => { setViewBeforeCompare((prev) => prev ?? view); @@ -932,6 +1241,27 @@ export function ChatPage(): ReactElement { completionTokens={contextUsage.completionTokens} /> ) : null} + {inferenceParams.checkpoint && activeGgufVariant ? ( + + ) : null} + + {endpointEnabled && ( + <> +
+
+ +
+ + {endpointBaseUrl !== endpointLocalUrl && ( + + )} +
+
+ {endpointExternalUrl && ( +
+ +
+ + {endpointBaseUrl !== endpointExternalUrl && ( + + )} +
+
+ )} +
+ +
+ + +
+
+
+ + +
+
+
+
+ + + +
+ {endpointApiTab === "completions" && ( +
+
+ + Python (OpenAI SDK) + + +
+
+ + cURL + + +
+
+ )} + {endpointApiTab === "responses" && ( +
+
+ + Python (OpenAI SDK) + + +
+
+ + cURL + + +
+
+ )} + {endpointApiTab === "messages" && ( +
+
+ + Python (Anthropic SDK) + + +
+
+ + cURL + + +
+
+ )} +
+ + )} + + );