From 46fcf98e74a489cd5c0ed38ca97f5c6c53a0bb99 Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Fri, 10 Apr 2026 10:09:19 +0000 Subject: [PATCH] feat: add Anthropic-compatible /v1/messages endpoint Proxy requests to llama-server's native Anthropic Messages API handler. Supports both streaming (SSE with message_start, content_block_delta, etc.) and non-streaming responses. Auth via x-api-key header (Anthropic SDK) or Authorization Bearer (OpenAI SDK). Adds Messages tab with Anthropic SDK and cURL snippets in the Access Endpoint dialog. --- studio/backend/auth/authentication.py | 31 ++++++++ studio/backend/routes/inference.py | 79 ++++++++++++++++++- .../frontend/src/features/chat/chat-page.tsx | 65 ++++++++++++++- 3 files changed, 171 insertions(+), 4 deletions(-) diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py index 4a18a72f9e..1a5f35e01a 100644 --- a/studio/backend/auth/authentication.py +++ b/studio/backend/auth/authentication.py @@ -197,3 +197,34 @@ async def get_current_subject_or_api_key( # Fall back to JWT validation (Studio frontend sessions) return await _get_current_subject(credentials, allow_password_change = False) + + +async def get_current_subject_or_api_key_anthropic( + request: Request, + credentials: Optional[HTTPAuthorizationCredentials] = Depends(_optional_security), +) -> str: + """Accept x-api-key header, Authorization: Bearer, or JWT. + + The Anthropic SDK sends ``x-api-key: `` instead of + ``Authorization: Bearer ``. This dependency checks both so that + the ``/v1/messages`` endpoint works with both Anthropic and OpenAI SDKs. + """ + external_key = getattr(request.app.state, "external_api_key", None) + + # Check x-api-key header first (Anthropic SDK default) + x_api_key = request.headers.get("x-api-key") + if x_api_key and external_key and x_api_key == external_key: + return "__api_user__" + + # Fall through to standard Bearer / JWT check + if credentials is None: + raise HTTPException( + status_code = status.HTTP_401_UNAUTHORIZED, + detail = "Missing Authorization or x-api-key header", + ) + + token = credentials.credentials + if external_key and token == external_key: + return "__api_user__" + + return await _get_current_subject(credentials, allow_password_change = False) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 36be56f05c..f503efc0d1 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -92,7 +92,7 @@ from models.inference import ( ResponsesInputMessage, ResponsesContentPart, ) -from auth.authentication import get_current_subject, get_current_subject_or_api_key +from auth.authentication import get_current_subject, get_current_subject_or_api_key, get_current_subject_or_api_key_anthropic import io import wave @@ -2235,6 +2235,83 @@ async def openai_responses( raise HTTPException(status_code = 500, detail = _friendly_error(e)) +# ===================================================================== +# Anthropic Messages API (/messages → /v1/messages) +# ===================================================================== + + +@router.post("/messages") +async def anthropic_messages( + request: Request, + current_subject: str = Depends(get_current_subject_or_api_key_anthropic), +): + """ + Anthropic-compatible Messages API endpoint. + + Proxies the request directly to llama-server's native ``/v1/messages`` + handler, which supports the full Anthropic Messages format including + streaming (``message_start``, ``content_block_delta``, etc.). + + Accepts auth via ``x-api-key`` header (Anthropic SDK) or + ``Authorization: Bearer`` (OpenAI SDK / general). + """ + import httpx + + llama_backend = get_llama_cpp_backend() + if not llama_backend.is_loaded: + raise HTTPException( + status_code = 400, + detail = "No GGUF model loaded. The Messages API requires a GGUF model.", + ) + + base_url = llama_backend._base_url + if not base_url: + raise HTTPException(status_code = 503, detail = "llama-server is not running") + + body = await request.body() + # Detect streaming from the JSON body + try: + payload = json.loads(body) + is_stream = payload.get("stream", False) + except Exception: + is_stream = False + + target_url = f"{base_url}/v1/messages" + + if is_stream: + + async def proxy_stream(): + async with httpx.AsyncClient() as client: + async with client.stream( + "POST", + target_url, + content = body, + headers = {"Content-Type": "application/json"}, + timeout = 300.0, + ) as resp: + async for chunk in resp.aiter_bytes(): + yield chunk + + return StreamingResponse( + proxy_stream(), + media_type = "text/event-stream", + headers = { + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + }, + ) + else: + async with httpx.AsyncClient() as client: + resp = await client.post( + target_url, + content = body, + headers = {"Content-Type": "application/json"}, + timeout = 300.0, + ) + return JSONResponse(content = resp.json(), status_code = resp.status_code) + + # OpenAI-Compatible Models Listing (/models → /v1/models) # ===================================================================== diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx index 445262638e..901adc02bb 100644 --- a/studio/frontend/src/features/chat/chat-page.tsx +++ b/studio/frontend/src/features/chat/chat-page.tsx @@ -690,7 +690,35 @@ print(response.output[0].content[0].text)`, }'`, [endpointApiKey, endpointBaseUrl, modelAlias], ); - const [endpointApiTab, setEndpointApiTab] = useState<"completions" | "responses">("completions"); + const anthropicPythonSnippet = useMemo( + () => `from anthropic import Anthropic + +client = Anthropic( + base_url="${endpointBaseUrl}", + api_key="${endpointApiKey}", +) + +message = client.messages.create( + model="${modelAlias}", + max_tokens=1024, + messages=[{"role": "user", "content": "What is 2+2?"}], +) + +print(message.content[0].text)`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const anthropicCurlSnippet = useMemo( + () => `curl ${endpointBaseUrl}/messages \\ + -H "Content-Type: application/json" \\ + -H "x-api-key: ${endpointApiKey}" \\ + -d '{ + "model": "${modelAlias}", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "What is 2+2?"}] + }'`, + [endpointApiKey, endpointBaseUrl, modelAlias], + ); + const [endpointApiTab, setEndpointApiTab] = useState<"completions" | "responses" | "messages">("completions"); const applyEndpointData = useCallback( (data: { @@ -1422,8 +1450,19 @@ print(response.output[0].content[0].text)`, > Responses + - {endpointApiTab === "completions" ? ( + {endpointApiTab === "completions" && (
@@ -1441,7 +1480,8 @@ print(response.output[0].content[0].text)`,
- ) : ( + )} + {endpointApiTab === "responses" && (
@@ -1460,6 +1500,25 @@ print(response.output[0].content[0].text)`,
)} + {endpointApiTab === "messages" && ( +
+
+ + Python (Anthropic SDK) + + +
+
+ + cURL + + +
+
+ )} )}