feat: add Anthropic-compatible /v1/messages endpoint
Proxy requests to llama-server's native Anthropic Messages API handler. Supports both streaming (SSE with message_start, content_block_delta, etc.) and non-streaming responses. Auth via x-api-key header (Anthropic SDK) or Authorization Bearer (OpenAI SDK). Adds Messages tab with Anthropic SDK and cURL snippets in the Access Endpoint dialog.
This commit is contained in:
Roland Tannous
parent
37f816d21d
commit
46fcf98e74
3 changed files with 171 additions and 4 deletions
|
|
@ -197,3 +197,34 @@ async def get_current_subject_or_api_key(
|
|||
|
||||
# Fall back to JWT validation (Studio frontend sessions)
|
||||
return await _get_current_subject(credentials, allow_password_change = False)
|
||||
|
||||
|
||||
async def get_current_subject_or_api_key_anthropic(
|
||||
request: Request,
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Depends(_optional_security),
|
||||
) -> str:
|
||||
"""Accept x-api-key header, Authorization: Bearer, or JWT.
|
||||
|
||||
The Anthropic SDK sends ``x-api-key: <key>`` instead of
|
||||
``Authorization: Bearer <key>``. This dependency checks both so that
|
||||
the ``/v1/messages`` endpoint works with both Anthropic and OpenAI SDKs.
|
||||
"""
|
||||
external_key = getattr(request.app.state, "external_api_key", None)
|
||||
|
||||
# Check x-api-key header first (Anthropic SDK default)
|
||||
x_api_key = request.headers.get("x-api-key")
|
||||
if x_api_key and external_key and x_api_key == external_key:
|
||||
return "__api_user__"
|
||||
|
||||
# Fall through to standard Bearer / JWT check
|
||||
if credentials is None:
|
||||
raise HTTPException(
|
||||
status_code = status.HTTP_401_UNAUTHORIZED,
|
||||
detail = "Missing Authorization or x-api-key header",
|
||||
)
|
||||
|
||||
token = credentials.credentials
|
||||
if external_key and token == external_key:
|
||||
return "__api_user__"
|
||||
|
||||
return await _get_current_subject(credentials, allow_password_change = False)
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ from models.inference import (
|
|||
ResponsesInputMessage,
|
||||
ResponsesContentPart,
|
||||
)
|
||||
from auth.authentication import get_current_subject, get_current_subject_or_api_key
|
||||
from auth.authentication import get_current_subject, get_current_subject_or_api_key, get_current_subject_or_api_key_anthropic
|
||||
|
||||
import io
|
||||
import wave
|
||||
|
|
@ -2235,6 +2235,83 @@ async def openai_responses(
|
|||
raise HTTPException(status_code = 500, detail = _friendly_error(e))
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# Anthropic Messages API (/messages → /v1/messages)
|
||||
# =====================================================================
|
||||
|
||||
|
||||
@router.post("/messages")
|
||||
async def anthropic_messages(
|
||||
request: Request,
|
||||
current_subject: str = Depends(get_current_subject_or_api_key_anthropic),
|
||||
):
|
||||
"""
|
||||
Anthropic-compatible Messages API endpoint.
|
||||
|
||||
Proxies the request directly to llama-server's native ``/v1/messages``
|
||||
handler, which supports the full Anthropic Messages format including
|
||||
streaming (``message_start``, ``content_block_delta``, etc.).
|
||||
|
||||
Accepts auth via ``x-api-key`` header (Anthropic SDK) or
|
||||
``Authorization: Bearer`` (OpenAI SDK / general).
|
||||
"""
|
||||
import httpx
|
||||
|
||||
llama_backend = get_llama_cpp_backend()
|
||||
if not llama_backend.is_loaded:
|
||||
raise HTTPException(
|
||||
status_code = 400,
|
||||
detail = "No GGUF model loaded. The Messages API requires a GGUF model.",
|
||||
)
|
||||
|
||||
base_url = llama_backend._base_url
|
||||
if not base_url:
|
||||
raise HTTPException(status_code = 503, detail = "llama-server is not running")
|
||||
|
||||
body = await request.body()
|
||||
# Detect streaming from the JSON body
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
is_stream = payload.get("stream", False)
|
||||
except Exception:
|
||||
is_stream = False
|
||||
|
||||
target_url = f"{base_url}/v1/messages"
|
||||
|
||||
if is_stream:
|
||||
|
||||
async def proxy_stream():
|
||||
async with httpx.AsyncClient() as client:
|
||||
async with client.stream(
|
||||
"POST",
|
||||
target_url,
|
||||
content = body,
|
||||
headers = {"Content-Type": "application/json"},
|
||||
timeout = 300.0,
|
||||
) as resp:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(
|
||||
proxy_stream(),
|
||||
media_type = "text/event-stream",
|
||||
headers = {
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
else:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
target_url,
|
||||
content = body,
|
||||
headers = {"Content-Type": "application/json"},
|
||||
timeout = 300.0,
|
||||
)
|
||||
return JSONResponse(content = resp.json(), status_code = resp.status_code)
|
||||
|
||||
|
||||
# OpenAI-Compatible Models Listing (/models → /v1/models)
|
||||
# =====================================================================
|
||||
|
||||
|
|
|
|||
|
|
@ -690,7 +690,35 @@ print(response.output[0].content[0].text)`,
|
|||
}'`,
|
||||
[endpointApiKey, endpointBaseUrl, modelAlias],
|
||||
);
|
||||
const [endpointApiTab, setEndpointApiTab] = useState<"completions" | "responses">("completions");
|
||||
const anthropicPythonSnippet = useMemo(
|
||||
() => `from anthropic import Anthropic
|
||||
|
||||
client = Anthropic(
|
||||
base_url="${endpointBaseUrl}",
|
||||
api_key="${endpointApiKey}",
|
||||
)
|
||||
|
||||
message = client.messages.create(
|
||||
model="${modelAlias}",
|
||||
max_tokens=1024,
|
||||
messages=[{"role": "user", "content": "What is 2+2?"}],
|
||||
)
|
||||
|
||||
print(message.content[0].text)`,
|
||||
[endpointApiKey, endpointBaseUrl, modelAlias],
|
||||
);
|
||||
const anthropicCurlSnippet = useMemo(
|
||||
() => `curl ${endpointBaseUrl}/messages \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-H "x-api-key: ${endpointApiKey}" \\
|
||||
-d '{
|
||||
"model": "${modelAlias}",
|
||||
"max_tokens": 1024,
|
||||
"messages": [{"role": "user", "content": "What is 2+2?"}]
|
||||
}'`,
|
||||
[endpointApiKey, endpointBaseUrl, modelAlias],
|
||||
);
|
||||
const [endpointApiTab, setEndpointApiTab] = useState<"completions" | "responses" | "messages">("completions");
|
||||
|
||||
const applyEndpointData = useCallback(
|
||||
(data: {
|
||||
|
|
@ -1422,8 +1450,19 @@ print(response.output[0].content[0].text)`,
|
|||
>
|
||||
Responses
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setEndpointApiTab("messages")}
|
||||
className={`flex-1 rounded px-2 py-1 text-xs font-medium transition-colors ${
|
||||
endpointApiTab === "messages"
|
||||
? "bg-background text-foreground shadow-sm"
|
||||
: "text-muted-foreground hover:text-foreground"
|
||||
}`}
|
||||
>
|
||||
Messages
|
||||
</button>
|
||||
</div>
|
||||
{endpointApiTab === "completions" ? (
|
||||
{endpointApiTab === "completions" && (
|
||||
<div className="space-y-2">
|
||||
<details className="rounded-md border p-2" open>
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
|
|
@ -1441,7 +1480,8 @@ print(response.output[0].content[0].text)`,
|
|||
<HighlightedSnippet language="bash" source={endpointCurlSnippet} />
|
||||
</details>
|
||||
</div>
|
||||
) : (
|
||||
)}
|
||||
{endpointApiTab === "responses" && (
|
||||
<div className="space-y-2">
|
||||
<details className="rounded-md border p-2" open>
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
|
|
@ -1460,6 +1500,25 @@ print(response.output[0].content[0].text)`,
|
|||
</details>
|
||||
</div>
|
||||
)}
|
||||
{endpointApiTab === "messages" && (
|
||||
<div className="space-y-2">
|
||||
<details className="rounded-md border p-2" open>
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
Python (Anthropic SDK)
|
||||
</summary>
|
||||
<HighlightedSnippet
|
||||
language="python"
|
||||
source={anthropicPythonSnippet}
|
||||
/>
|
||||
</details>
|
||||
<details className="rounded-md border p-2">
|
||||
<summary className="cursor-pointer text-xs font-medium">
|
||||
cURL
|
||||
</summary>
|
||||
<HighlightedSnippet language="bash" source={anthropicCurlSnippet} />
|
||||
</details>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue