studio: only open the API monitor for real API clients
CI caught this: the Chat UI Playwright run failed because the floating panel opened during an ordinary chat turn and its Expand button covered the composer's Send button. The panel opened because Studio's own chat goes through the same tracked endpoints as the OpenAI-compatible API, so any conversation looked like API traffic. That is wrong regardless of the click interception: this panel exists for when Unsloth is being used as an API server, not when someone is using Unsloth. Record on each monitor entry whether the caller authenticated with an sk-unsloth key rather than a UI session, and auto-open only for those. The discriminator already existed for other routes; this reuses it.
This commit is contained in:
parent
8f322aa627
commit
0a49dfd047
6 changed files with 94 additions and 5 deletions
|
|
@ -41,6 +41,10 @@ class ApiMonitorEntry:
|
|||
started_at: float
|
||||
updated_at: float
|
||||
subject: Optional[str] = None
|
||||
# True when the caller used an sk-unsloth key rather than a UI session. The
|
||||
# floating panel only opens itself for these: Studio's own chat goes through
|
||||
# the same endpoints, and popping the monitor open mid-chat is noise.
|
||||
via_api_key: bool = False
|
||||
# Monotonic anchors so duration math survives wall-clock steps (NTP).
|
||||
started_monotonic: float = 0.0
|
||||
finished_monotonic: Optional[float] = None
|
||||
|
|
@ -70,6 +74,7 @@ class ApiMonitorEntry:
|
|||
"endpoint": self.endpoint,
|
||||
"method": self.method,
|
||||
"model": self.model,
|
||||
"via_api_key": self.via_api_key,
|
||||
"prompt_preview": _trim(self.prompt, _PREVIEW_CHARS),
|
||||
"reply_preview": _trim(self.reply, _PREVIEW_CHARS),
|
||||
"prompt_truncated": len(self.prompt) > _PREVIEW_CHARS,
|
||||
|
|
@ -107,6 +112,7 @@ class ApiMonitor:
|
|||
prompt: str,
|
||||
context_length: Optional[int] = None,
|
||||
subject: Optional[str] = None,
|
||||
via_api_key: bool = False,
|
||||
) -> str:
|
||||
now = time.time()
|
||||
entry = ApiMonitorEntry(
|
||||
|
|
@ -121,6 +127,7 @@ class ApiMonitor:
|
|||
started_at = now,
|
||||
updated_at = now,
|
||||
subject = subject,
|
||||
via_api_key = via_api_key,
|
||||
started_monotonic = time.monotonic(),
|
||||
context_length = context_length,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1754,7 +1754,24 @@ from core.inference.anthropic_compat import (
|
|||
AnthropicStreamEmitter,
|
||||
AnthropicPassthroughEmitter,
|
||||
)
|
||||
from auth.authentication import get_current_subject
|
||||
from auth.authentication import API_KEY_PREFIX, get_current_subject
|
||||
|
||||
|
||||
def _request_used_api_key(request: Any) -> bool:
|
||||
"""True when this request authenticated with an sk-unsloth key.
|
||||
|
||||
Studio's own chat hits these same endpoints with a session JWT, so this is
|
||||
what separates "someone is using Unsloth as an API server" from "someone is
|
||||
using Unsloth".
|
||||
"""
|
||||
try:
|
||||
header = request.headers.get("authorization") or ""
|
||||
except Exception:
|
||||
return False
|
||||
scheme, _, token = header.partition(" ")
|
||||
return scheme.lower() == "bearer" and token.startswith(API_KEY_PREFIX)
|
||||
|
||||
|
||||
from state.tool_approvals import resolve_tool_decision
|
||||
|
||||
from core.inference.key_exchange import decrypt_api_key
|
||||
|
|
@ -7238,6 +7255,7 @@ async def _proxy_to_external_provider(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -7798,6 +7816,7 @@ async def openai_chat_completions(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
tts_monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model_label,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -7865,6 +7884,7 @@ async def openai_chat_completions(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model_name,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -8009,6 +8029,7 @@ async def openai_chat_completions(
|
|||
if monitor_id is None and not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = model_name,
|
||||
prompt = _monitor_prompt_from_messages(payload.messages),
|
||||
|
|
@ -10832,6 +10853,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
|
|||
prompt_text = _flatten_monitor_prompt(body.get("prompt", ""))
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"),
|
||||
prompt = prompt_text,
|
||||
|
|
@ -11043,6 +11065,7 @@ async def openai_embeddings(request: Request, current_subject: str = Depends(get
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = str(body.get("model") or _llama_public_model_id(llama_backend) or "default"),
|
||||
prompt = prompt_text,
|
||||
|
|
@ -11637,6 +11660,7 @@ async def _responses_non_streaming(
|
|||
monitor_id = api_monitor.start(
|
||||
endpoint = getattr(getattr(request, "url", None), "path", "/v1/responses"),
|
||||
method = getattr(request, "method", "POST"),
|
||||
via_api_key = _request_used_api_key(request),
|
||||
model = payload.model,
|
||||
prompt = _monitor_prompt_from_messages(messages),
|
||||
context_length = _monitor_context_length(),
|
||||
|
|
@ -12828,6 +12852,7 @@ async def openai_responses(
|
|||
if not getattr(request.state, "skip_api_monitor", False):
|
||||
monitor_id = api_monitor.start(
|
||||
endpoint = request.url.path,
|
||||
via_api_key = _request_used_api_key(request),
|
||||
method = request.method,
|
||||
model = payload.model,
|
||||
prompt = _monitor_prompt_from_messages(messages),
|
||||
|
|
@ -13296,6 +13321,7 @@ async def anthropic_messages(
|
|||
monitor_id = api_monitor.start(
|
||||
endpoint = getattr(request_url, "path", "/v1/messages"),
|
||||
method = getattr(request, "method", "POST"),
|
||||
via_api_key = _request_used_api_key(request),
|
||||
model = model_name,
|
||||
prompt = _monitor_prompt_from_messages(openai_messages),
|
||||
context_length = monitor_context_length,
|
||||
|
|
|
|||
|
|
@ -289,3 +289,28 @@ def test_api_monitor_clear_is_scoped_to_one_subject():
|
|||
# Passing no subject is the explicit "everything" path.
|
||||
monitor.clear()
|
||||
assert monitor.snapshot(subject = "bob") == []
|
||||
|
||||
|
||||
def test_api_monitor_records_whether_the_caller_used_an_api_key():
|
||||
# Studio's own chat hits these endpoints with a session JWT. The floating
|
||||
# panel keys its auto-open off this flag, so mislabelling in-app chat as API
|
||||
# traffic pops the panel over the composer mid-conversation.
|
||||
monitor = ApiMonitor(max_entries = 4)
|
||||
ui = monitor.start(
|
||||
endpoint = "/api/inference/chat",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "hi",
|
||||
subject = "u",
|
||||
)
|
||||
api = monitor.start(
|
||||
endpoint = "/v1/chat/completions",
|
||||
method = "POST",
|
||||
model = "m",
|
||||
prompt = "hi",
|
||||
subject = "u",
|
||||
via_api_key = True,
|
||||
)
|
||||
by_id = {entry["id"]: entry for entry in monitor.snapshot(subject = "u")}
|
||||
assert by_id[ui]["via_api_key"] is False
|
||||
assert by_id[api]["via_api_key"] is True
|
||||
|
|
|
|||
|
|
@ -4247,3 +4247,20 @@ def test_ambiguous_case_fallback_matches_nothing(monkeypatch):
|
|||
settings.set_model_override("/models/foo.gguf", max_seq_length = 1024)
|
||||
settings.set_model_override("/models/FOO.gguf", max_seq_length = 8192)
|
||||
assert settings.get_model_override("/models/Foo.gguf") == {}
|
||||
|
||||
|
||||
def test_request_used_api_key_distinguishes_key_from_session():
|
||||
from auth.authentication import API_KEY_PREFIX
|
||||
|
||||
class _Req:
|
||||
def __init__(self, header):
|
||||
self.headers = {"authorization": header} if header else {}
|
||||
|
||||
assert inference_route._request_used_api_key(_Req(f"Bearer {API_KEY_PREFIX}abc")) is True
|
||||
assert inference_route._request_used_api_key(_Req(f"bearer {API_KEY_PREFIX}abc")) is True
|
||||
assert inference_route._request_used_api_key(_Req("Bearer eyJhbGciOiJIUzI1NiJ9.x")) is False
|
||||
assert inference_route._request_used_api_key(_Req("")) is False
|
||||
assert inference_route._request_used_api_key(_Req(None)) is False
|
||||
# A malformed request object must read as "not an API key", never raise, since
|
||||
# this runs on the hot path of every tracked request.
|
||||
assert inference_route._request_used_api_key(object()) is False
|
||||
|
|
|
|||
|
|
@ -172,7 +172,12 @@ export function ApiMonitorOverlay(): ReactElement | null {
|
|||
return;
|
||||
}
|
||||
const seen = seenIdsRef.current;
|
||||
const hasNewTraffic = ids.some((id) => !seen.has(id));
|
||||
// Only API-key traffic counts. Studio's own chat goes through these same
|
||||
// endpoints, and this panel is about serving other clients, not about the
|
||||
// request the user is watching stream in front of them.
|
||||
const hasNewTraffic = data.entries.some(
|
||||
(entry) => entry.via_api_key && !seen.has(entry.id),
|
||||
);
|
||||
// Re-seed each poll so the set stays bounded by the server's ring buffer.
|
||||
seenIdsRef.current = new Set(ids);
|
||||
if (!hasNewTraffic) {
|
||||
|
|
|
|||
|
|
@ -169,7 +169,10 @@ export interface LoadModelResponse {
|
|||
max_context_length?: number | null;
|
||||
native_context_length?: number | null;
|
||||
supports_reasoning?: boolean;
|
||||
reasoning_style?: "enable_thinking" | "reasoning_effort" | "enable_thinking_effort";
|
||||
reasoning_style?:
|
||||
| "enable_thinking"
|
||||
| "reasoning_effort"
|
||||
| "enable_thinking_effort";
|
||||
reasoning_effort_levels?: string[];
|
||||
reasoning_always_on?: boolean;
|
||||
supports_preserve_thinking?: boolean;
|
||||
|
|
@ -220,7 +223,10 @@ export interface InferenceStatusResponse {
|
|||
} | null;
|
||||
requires_trust_remote_code?: boolean;
|
||||
supports_reasoning?: boolean;
|
||||
reasoning_style?: "enable_thinking" | "reasoning_effort" | "enable_thinking_effort";
|
||||
reasoning_style?:
|
||||
| "enable_thinking"
|
||||
| "reasoning_effort"
|
||||
| "enable_thinking_effort";
|
||||
reasoning_effort_levels?: string[];
|
||||
reasoning_always_on?: boolean;
|
||||
supports_preserve_thinking?: boolean;
|
||||
|
|
@ -268,6 +274,9 @@ export interface ApiMonitorEntry {
|
|||
model: string;
|
||||
prompt?: string;
|
||||
reply?: string;
|
||||
// True when the caller used an API key rather than a UI session. The floating
|
||||
// panel keys its auto-open off this so Studio's own chat does not pop it.
|
||||
via_api_key: boolean;
|
||||
prompt_preview: string;
|
||||
reply_preview: string;
|
||||
prompt_truncated: boolean;
|
||||
|
|
@ -389,7 +398,7 @@ export interface OpenAIChatCompletionsRequest {
|
|||
| "xhigh"
|
||||
| null;
|
||||
preserve_thinking?: boolean | null;
|
||||
thinking?: {type: "disabled" | "enabled";} | null;
|
||||
thinking?: { type: "disabled" | "enabled" } | null;
|
||||
enable_tools?: boolean | null;
|
||||
enabled_tools?: string[];
|
||||
/** Local models + enable_tools only. */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue