Studio: out-of-band refusal signal + fast-mode prefix/usage/pricing fixes
The text sentinel for the Anthropic refusal drop signal was spoofable: any assistant message containing the literal <!--studio:anthropic-refusal--> would prune the prior user + assistant pair on the next request. Move the signal onto a separate _toolEvent chunk that the chat adapter latches into assistant.metadata.custom.anthropicRefusal; assistant text can no longer control the pruner. Tighten the fast-mode model gate (backend + frontend) to require a "-" family boundary so claude-opus-4-70 / claude-opus-4-7b style IDs do not get speed: "fast" on a naive startswith match. Use survivingMessages for the image / audio attachment scan so a refused user turn does not gate or mis-attribute the next non-refused turn. Propagate Anthropic usage.speed onto the OpenAI-style usage chunk and apply the documented 6x fast-mode multiplier in the cost calculator (stacks with prompt-cache multipliers per the docs); expose the new multiplier on the pricing snapshot for the UI tooltip. Tests cover the tool-event chunk shape, the prefix-collision rejects, usage.speed propagation, the 6x pricing math, and that the visible refusal text carries no embedded sentinel.
This commit is contained in:
parent
41531cdbe2
commit
4f1afdb535
7 changed files with 302 additions and 48 deletions
|
|
@ -191,7 +191,15 @@ def _anthropic_supports_compaction(model: str) -> bool:
|
||||||
|
|
||||||
|
|
||||||
def _anthropic_supports_fast_mode(model: str) -> bool:
|
def _anthropic_supports_fast_mode(model: str) -> bool:
|
||||||
return model.startswith(_ANTHROPIC_FAST_MODE_PREFIXES)
|
# Require the prefix to terminate at a family boundary (end of
|
||||||
|
# string or a "-" separator before the date snapshot) so the
|
||||||
|
# check does not light up on hypothetical IDs like
|
||||||
|
# "claude-opus-4-70" / "claude-opus-4-7b" that merely share a
|
||||||
|
# prefix with the supported families.
|
||||||
|
return any(
|
||||||
|
model == p or model.startswith(f"{p}-")
|
||||||
|
for p in _ANTHROPIC_FAST_MODE_PREFIXES
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class _MistralThinkingSpec(NamedTuple):
|
class _MistralThinkingSpec(NamedTuple):
|
||||||
|
|
@ -2466,20 +2474,27 @@ class ExternalProviderClient:
|
||||||
"Anthropic refusal stop_reason (model=%s)",
|
"Anthropic refusal stop_reason (model=%s)",
|
||||||
model,
|
model,
|
||||||
)
|
)
|
||||||
# Trailing HTML comment is a stable
|
# User-facing notice so the chat bubble
|
||||||
# sentinel the chat-adapter matches
|
# is not silently empty after the
|
||||||
# to drop this assistant turn from
|
# safety classifier truncates the
|
||||||
# the next request's outbound
|
# response. The out-of-band drop
|
||||||
# history. Anthropic's docs say
|
# signal rides a separate _toolEvent
|
||||||
# the refused turn must be removed
|
# below so assistant text can never
|
||||||
# or updated before continuing or
|
# spoof a context reset by including
|
||||||
# the next call will keep refusing.
|
# a literal marker. Anthropic's docs
|
||||||
|
# say the refused turn must be
|
||||||
|
# removed or updated before
|
||||||
|
# continuing or the next call will
|
||||||
|
# keep refusing.
|
||||||
|
# https://platform.claude.com/docs/en/test-and-evaluate/strengthen-guardrails/handle-streaming-refusals
|
||||||
yield _content_chunk(
|
yield _content_chunk(
|
||||||
"\n\n_The response was stopped by "
|
"\n\n_The response was stopped by "
|
||||||
"Anthropic's safety classifier. Edit "
|
"Anthropic's safety classifier. Edit "
|
||||||
"or remove the previous turn and try "
|
"or remove the previous turn and try "
|
||||||
"again._"
|
"again._"
|
||||||
"\n<!--studio:anthropic-refusal-->"
|
)
|
||||||
|
yield _emit_tool_event(
|
||||||
|
{"type": "anthropic_refusal"}
|
||||||
)
|
)
|
||||||
if mapped is not None:
|
if mapped is not None:
|
||||||
chunk = {
|
chunk = {
|
||||||
|
|
@ -4010,6 +4025,15 @@ def _build_usage_chunk(
|
||||||
"cache_creation_input_tokens": cache_creation,
|
"cache_creation_input_tokens": cache_creation,
|
||||||
"cache_read_input_tokens": cache_read,
|
"cache_read_input_tokens": cache_read,
|
||||||
}
|
}
|
||||||
|
# Anthropic fast-mode responses include `usage.speed` so callers
|
||||||
|
# can verify whether a premium fast-mode request actually ran
|
||||||
|
# fast (the API silently falls back to "standard" when the
|
||||||
|
# beta is unsupported or rate-limited). Surface it on the
|
||||||
|
# OpenAI-style chunk so the pricing/cost ledger can apply the
|
||||||
|
# 6x multiplier without re-derivation.
|
||||||
|
speed = last_usage.get("speed")
|
||||||
|
if speed in ("fast", "standard"):
|
||||||
|
usage_block["speed"] = speed
|
||||||
else:
|
else:
|
||||||
prompt_tokens = last_usage.get("input_tokens") or 0
|
prompt_tokens = last_usage.get("input_tokens") or 0
|
||||||
cached = 0
|
cached = 0
|
||||||
|
|
|
||||||
|
|
@ -105,6 +105,10 @@ OPENAI_PRICING: dict[str, dict[str, float]] = {
|
||||||
ANTHROPIC_CACHE_5M_WRITE_MULT = 1.25
|
ANTHROPIC_CACHE_5M_WRITE_MULT = 1.25
|
||||||
ANTHROPIC_CACHE_1H_WRITE_MULT = 2.0
|
ANTHROPIC_CACHE_1H_WRITE_MULT = 2.0
|
||||||
ANTHROPIC_CACHE_READ_MULT = 0.1
|
ANTHROPIC_CACHE_READ_MULT = 0.1
|
||||||
|
# Anthropic fast-mode beta (Opus 4.6 / 4.7 only): 6x standard rates on
|
||||||
|
# both input and output across the full context window.
|
||||||
|
# https://platform.claude.com/docs/en/build-with-claude/fast-mode#pricing
|
||||||
|
ANTHROPIC_FAST_MODE_MULT = 6.0
|
||||||
|
|
||||||
# OpenAI: cache reads are 0.1x base input, cache writes are not billed
|
# OpenAI: cache reads are 0.1x base input, cache writes are not billed
|
||||||
# separately (the first prefix-write request just pays normal input).
|
# separately (the first prefix-write request just pays normal input).
|
||||||
|
|
@ -235,6 +239,17 @@ def calculate_cost(
|
||||||
base = prices["input_per_mtok"]
|
base = prices["input_per_mtok"]
|
||||||
out_per = prices["output_per_mtok"]
|
out_per = prices["output_per_mtok"]
|
||||||
|
|
||||||
|
# Anthropic fast-mode beta: 6x standard on both input + output across
|
||||||
|
# the full context window. Prompt-cache multipliers stack on top of
|
||||||
|
# the fast-mode base per the docs, so applying the multiplier once
|
||||||
|
# to (base, out_per) propagates correctly into the cache_*_usd
|
||||||
|
# buckets computed below.
|
||||||
|
if provider == "anthropic" and usage.get("speed") == "fast":
|
||||||
|
base *= ANTHROPIC_FAST_MODE_MULT
|
||||||
|
out_per *= ANTHROPIC_FAST_MODE_MULT
|
||||||
|
if out["model_priced"]:
|
||||||
|
out["model_priced"] = f"{out['model_priced']} (fast)"
|
||||||
|
|
||||||
out["input_usd"] = (input_tokens / 1_000_000.0) * base
|
out["input_usd"] = (input_tokens / 1_000_000.0) * base
|
||||||
out["output_usd"] = (output_tokens / 1_000_000.0) * out_per
|
out["output_usd"] = (output_tokens / 1_000_000.0) * out_per
|
||||||
|
|
||||||
|
|
@ -315,6 +330,7 @@ def pricing_snapshot() -> dict[str, Any]:
|
||||||
"cache_5m_write_mult": ANTHROPIC_CACHE_5M_WRITE_MULT,
|
"cache_5m_write_mult": ANTHROPIC_CACHE_5M_WRITE_MULT,
|
||||||
"cache_1h_write_mult": ANTHROPIC_CACHE_1H_WRITE_MULT,
|
"cache_1h_write_mult": ANTHROPIC_CACHE_1H_WRITE_MULT,
|
||||||
"cache_read_mult": ANTHROPIC_CACHE_READ_MULT,
|
"cache_read_mult": ANTHROPIC_CACHE_READ_MULT,
|
||||||
|
"fast_mode_mult": ANTHROPIC_FAST_MODE_MULT,
|
||||||
"web_search_usd_per_1k": ANTHROPIC_WEB_SEARCH_USD_PER_1K,
|
"web_search_usd_per_1k": ANTHROPIC_WEB_SEARCH_USD_PER_1K,
|
||||||
"code_execution_usd_per_hour": ANTHROPIC_CODE_EXEC_USD_PER_HOUR,
|
"code_execution_usd_per_hour": ANTHROPIC_CODE_EXEC_USD_PER_HOUR,
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -160,14 +160,20 @@ def test_refusal_emits_user_facing_notice_and_content_filter_finish(monkeypatch)
|
||||||
assert "Hello." in body, body
|
assert "Hello." in body, body
|
||||||
|
|
||||||
|
|
||||||
def test_refusal_emits_hidden_sentinel_for_chat_adapter_drop(monkeypatch):
|
def test_refusal_emits_tool_event_for_chat_adapter_drop(monkeypatch):
|
||||||
"""Refused turns embed a hidden sentinel the chat-adapter matches on.
|
"""Refused turns emit an out-of-band `_toolEvent` the chat-adapter
|
||||||
|
latches into assistant `metadata.custom.anthropicRefusal`.
|
||||||
|
|
||||||
The frontend drops any assistant message containing this sentinel
|
The frontend drops any assistant message with that metadata flag
|
||||||
from the next request's outbound history, per Anthropic's docs:
|
from the next request's outbound history, per Anthropic's docs:
|
||||||
leaving the refused output in context causes the next call to keep
|
leaving the refused output in context causes the next call to keep
|
||||||
refusing.
|
refusing. Using a tool event (not a text sentinel) means assistant
|
||||||
|
content can never spoof a context reset.
|
||||||
"""
|
"""
|
||||||
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
||||||
body = "\n".join(lines)
|
body = "\n".join(lines)
|
||||||
assert "studio:anthropic-refusal" in body, body
|
assert '"_toolEvent": {"type": "anthropic_refusal"}' in body, body
|
||||||
|
# The visible refusal notice must NOT carry a text sentinel: an
|
||||||
|
# assistant message that echoes that string would otherwise spoof
|
||||||
|
# a context reset on the next request.
|
||||||
|
assert "studio:anthropic-refusal" not in body, body
|
||||||
|
|
|
||||||
|
|
@ -23,14 +23,16 @@ This file fills in the remaining behaviour cliffs:
|
||||||
duplicates and no truncation.
|
duplicates and no truncation.
|
||||||
* Idempotence: setting ``fast_mode=True`` twice via the same body still
|
* Idempotence: setting ``fast_mode=True`` twice via the same body still
|
||||||
results in one beta-header entry.
|
results in one beta-header entry.
|
||||||
* Streaming refusal sentinel: emitted exactly once, with the exact
|
* Streaming refusal signal: a single out-of-band ``_toolEvent`` carrying
|
||||||
``<!--studio:anthropic-refusal-->`` token the frontend matches on,
|
``{"type": "anthropic_refusal"}`` rides alongside the visible refusal
|
||||||
and the notice always precedes the finish_reason chunk so a UI
|
notice. The frontend latches the tool event into assistant
|
||||||
|
metadata.custom.anthropicRefusal; assistant text never controls the
|
||||||
|
pruner. The notice always precedes the finish_reason chunk so a UI
|
||||||
reading the SSE in order paints text before flipping to
|
reading the SSE in order paints text before flipping to
|
||||||
``content_filter``.
|
``content_filter``.
|
||||||
* Refusal on a non-Opus model: refusal handling is provider-side, not
|
* Refusal on a non-Opus model: refusal handling is provider-side, not
|
||||||
model-gated, so a refusal mid-stream on Sonnet must still surface the
|
model-gated, so a refusal mid-stream on Sonnet must still surface the
|
||||||
notice + sentinel.
|
notice + tool event.
|
||||||
* Non-destruction: when ``fast_mode`` is ``None``, the outbound body
|
* Non-destruction: when ``fast_mode`` is ``None``, the outbound body
|
||||||
and headers must be byte-identical to the version that omits the
|
and headers must be byte-identical to the version that omits the
|
||||||
argument entirely. This guarantees the upgrade path is non-breaking
|
argument entirely. This guarantees the upgrade path is non-breaking
|
||||||
|
|
@ -328,30 +330,41 @@ def test_refusal_notice_appears_before_content_filter_chunk(monkeypatch):
|
||||||
assert notice_idx < filter_idx, (notice_idx, filter_idx, lines)
|
assert notice_idx < filter_idx, (notice_idx, filter_idx, lines)
|
||||||
|
|
||||||
|
|
||||||
def test_refusal_sentinel_emitted_exactly_once(monkeypatch):
|
def test_refusal_tool_event_emitted_exactly_once(monkeypatch):
|
||||||
"""A single refusal must emit the chat-adapter drop sentinel one time.
|
"""A single refusal must emit the chat-adapter drop signal one time.
|
||||||
|
|
||||||
The frontend's ``toOpenAIMessage`` uses ``includes`` for the match,
|
The frontend latches the tool event into assistant metadata and
|
||||||
so duplicates wouldn't cause false drops -- but emitting it twice
|
uses it to drop the refused pair from the next request. Emitting
|
||||||
would still inflate the bubble and break the UX guarantee that the
|
twice would still be metadata-idempotent but indicates a backend
|
||||||
sentinel is an invisible HTML comment.
|
bug, so pin the count.
|
||||||
"""
|
"""
|
||||||
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
||||||
body = "\n".join(lines)
|
body = "\n".join(lines)
|
||||||
count = body.count("<!--studio:anthropic-refusal-->")
|
count = body.count('"_toolEvent": {"type": "anthropic_refusal"}')
|
||||||
assert count == 1, (count, body)
|
assert count == 1, (count, body)
|
||||||
|
|
||||||
|
|
||||||
|
def test_refusal_text_carries_no_html_sentinel(monkeypatch):
|
||||||
|
"""Belt-and-braces: ensure the assistant-visible refusal text does
|
||||||
|
not embed any ``studio:anthropic-refusal`` marker. The drop signal
|
||||||
|
must ride the out-of-band ``_toolEvent`` channel only -- otherwise
|
||||||
|
an assistant message that echoes the literal marker would spoof a
|
||||||
|
context reset on the next request."""
|
||||||
|
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
||||||
|
body = "\n".join(lines)
|
||||||
|
assert "studio:anthropic-refusal" not in body, body
|
||||||
|
|
||||||
|
|
||||||
def test_refusal_handling_works_on_sonnet_model(monkeypatch):
|
def test_refusal_handling_works_on_sonnet_model(monkeypatch):
|
||||||
"""Refusal handling is provider-side, not gated on a fast-mode-capable
|
"""Refusal handling is provider-side, not gated on a fast-mode-capable
|
||||||
model. If Anthropic's classifier refuses a Sonnet stream it must
|
model. If Anthropic's classifier refuses a Sonnet stream it must
|
||||||
surface the same notice + sentinel + content_filter mapping."""
|
surface the same notice + tool event + content_filter mapping."""
|
||||||
_, lines = _capture(
|
_, lines = _capture(
|
||||||
monkeypatch, sse = _refusal_sse("claude-sonnet-4-6"), model = "claude-sonnet-4-6"
|
monkeypatch, sse = _refusal_sse("claude-sonnet-4-6"), model = "claude-sonnet-4-6"
|
||||||
)
|
)
|
||||||
body = "\n".join(lines)
|
body = "\n".join(lines)
|
||||||
assert "stopped by Anthropic's safety classifier" in body, body
|
assert "stopped by Anthropic's safety classifier" in body, body
|
||||||
assert "<!--studio:anthropic-refusal-->" in body, body
|
assert '"_toolEvent": {"type": "anthropic_refusal"}' in body, body
|
||||||
assert '"finish_reason": "content_filter"' in body, body
|
assert '"finish_reason": "content_filter"' in body, body
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -369,7 +382,8 @@ def test_refusal_preserves_partial_assistant_text(monkeypatch):
|
||||||
def test_refusal_chunk_is_proper_openai_delta_shape(monkeypatch):
|
def test_refusal_chunk_is_proper_openai_delta_shape(monkeypatch):
|
||||||
"""The notice rides a normal ``choices[0].delta.content`` chunk, not
|
"""The notice rides a normal ``choices[0].delta.content`` chunk, not
|
||||||
a finish_reason chunk, so OpenAI-spec clients (Aurora, OpenAI SDK)
|
a finish_reason chunk, so OpenAI-spec clients (Aurora, OpenAI SDK)
|
||||||
treat it as ordinary streamed text."""
|
treat it as ordinary streamed text. The drop signal arrives on a
|
||||||
|
separate `_toolEvent` chunk (verified below)."""
|
||||||
_, lines = _capture(monkeypatch, sse = _refusal_sse(), model = "claude-opus-4-7")
|
_, lines = _capture(monkeypatch, sse = _refusal_sse(), model = "claude-opus-4-7")
|
||||||
# Find the chunk that carries the refusal text.
|
# Find the chunk that carries the refusal text.
|
||||||
notice_chunk = None
|
notice_chunk = None
|
||||||
|
|
@ -383,7 +397,27 @@ def test_refusal_chunk_is_proper_openai_delta_shape(monkeypatch):
|
||||||
# Must NOT carry a finish_reason itself -- that comes on the next
|
# Must NOT carry a finish_reason itself -- that comes on the next
|
||||||
# chunk.
|
# chunk.
|
||||||
assert choice.get("finish_reason") in (None,), notice_chunk
|
assert choice.get("finish_reason") in (None,), notice_chunk
|
||||||
assert "<!--studio:anthropic-refusal-->" in choice["delta"]["content"]
|
# Refusal text is plain-spoken; no embedded sentinel.
|
||||||
|
assert "studio:anthropic-refusal" not in choice["delta"]["content"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_refusal_tool_event_chunk_shape(monkeypatch):
|
||||||
|
"""The out-of-band drop signal rides a separate chunk shaped like a
|
||||||
|
Studio `_toolEvent` envelope (choices=[{index:0, delta:{},
|
||||||
|
finish_reason:null}] + `_toolEvent`). The frontend latches on
|
||||||
|
`_toolEvent.type == "anthropic_refusal"` and stamps the assistant
|
||||||
|
message metadata; assistant text never controls the prune."""
|
||||||
|
_, lines = _capture(monkeypatch, sse = _refusal_sse(), model = "claude-opus-4-7")
|
||||||
|
refusal_chunk = None
|
||||||
|
for line in lines:
|
||||||
|
if line.startswith("data: ") and "anthropic_refusal" in line:
|
||||||
|
refusal_chunk = json.loads(line[len("data: ") :])
|
||||||
|
break
|
||||||
|
assert refusal_chunk is not None, lines
|
||||||
|
assert refusal_chunk["_toolEvent"] == {"type": "anthropic_refusal"}, refusal_chunk
|
||||||
|
choice = refusal_chunk["choices"][0]
|
||||||
|
assert choice["delta"] == {}, refusal_chunk
|
||||||
|
assert choice["finish_reason"] is None, refusal_chunk
|
||||||
|
|
||||||
|
|
||||||
# ──────────────────────────── future-proofing ────────────────────────────
|
# ──────────────────────────── future-proofing ────────────────────────────
|
||||||
|
|
@ -415,3 +449,85 @@ def test_fast_mode_dropped_on_opus_4_5_dated_snapshot(monkeypatch):
|
||||||
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-5-2025-08-01")
|
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-5-2025-08-01")
|
||||||
assert "speed" not in cap["body"], cap["body"]
|
assert "speed" not in cap["body"], cap["body"]
|
||||||
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fast_mode_rejects_prefix_collision_4_70(monkeypatch):
|
||||||
|
"""The family gate must require a "-" boundary after the supported
|
||||||
|
prefix so hypothetical IDs like ``claude-opus-4-70`` or
|
||||||
|
``claude-opus-4-7b`` do not get fast-mode on a naive
|
||||||
|
``startswith`` match."""
|
||||||
|
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-70")
|
||||||
|
assert "speed" not in cap["body"], cap["body"]
|
||||||
|
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fast_mode_rejects_prefix_collision_4_7b(monkeypatch):
|
||||||
|
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-7b")
|
||||||
|
assert "speed" not in cap["body"], cap["body"]
|
||||||
|
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fast_mode_rejects_prefix_collision_4_6_extra(monkeypatch):
|
||||||
|
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-60")
|
||||||
|
assert "speed" not in cap["body"], cap["body"]
|
||||||
|
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||||
|
|
||||||
|
|
||||||
|
# ──────────────────────────── usage.speed propagation ────────────────────────────
|
||||||
|
def _fast_speed_sse(model: str = "claude-opus-4-7", speed: str = "fast") -> bytes:
|
||||||
|
return (
|
||||||
|
b'event: message_start\ndata: {"type":"message_start","message":'
|
||||||
|
b'{"id":"m1","content":[],"model":"' + model.encode() + b'",'
|
||||||
|
b'"role":"assistant","stop_reason":null,"usage":'
|
||||||
|
b'{"input_tokens":4,"output_tokens":1}}}\n\n'
|
||||||
|
b'event: content_block_start\ndata: {"type":"content_block_start",'
|
||||||
|
b'"index":0,"content_block":{"type":"text","text":""}}\n\n'
|
||||||
|
b'event: content_block_delta\ndata: {"type":"content_block_delta",'
|
||||||
|
b'"index":0,"delta":{"type":"text_delta","text":"hi"}}\n\n'
|
||||||
|
b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n'
|
||||||
|
b'event: message_delta\ndata: {"type":"message_delta",'
|
||||||
|
b'"delta":{"stop_reason":"end_turn"},'
|
||||||
|
b'"usage":{"output_tokens":5,"speed":"' + speed.encode() + b'"}}\n\n'
|
||||||
|
b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_usage_speed_propagates_to_final_usage_chunk_fast(monkeypatch):
|
||||||
|
"""When Anthropic returns ``usage.speed == "fast"`` the Studio
|
||||||
|
OpenAI-style usage chunk must carry that field so the cost ledger
|
||||||
|
can apply the 6x multiplier and clients can verify a fast-mode
|
||||||
|
request actually ran fast."""
|
||||||
|
_, lines = _capture(monkeypatch, sse = _fast_speed_sse(speed = "fast"))
|
||||||
|
usage_lines = [
|
||||||
|
l for l in lines if l.startswith("data: ") and '"usage"' in l
|
||||||
|
]
|
||||||
|
assert usage_lines, lines
|
||||||
|
parsed = [json.loads(l[len("data: ") :]) for l in usage_lines]
|
||||||
|
speeds = [p["usage"].get("speed") for p in parsed if "usage" in p]
|
||||||
|
assert "fast" in speeds, parsed
|
||||||
|
|
||||||
|
|
||||||
|
def test_usage_speed_propagates_to_final_usage_chunk_standard(monkeypatch):
|
||||||
|
_, lines = _capture(monkeypatch, sse = _fast_speed_sse(speed = "standard"))
|
||||||
|
parsed = [
|
||||||
|
json.loads(l[len("data: ") :])
|
||||||
|
for l in lines
|
||||||
|
if l.startswith("data: ") and '"usage"' in l
|
||||||
|
]
|
||||||
|
speeds = [p["usage"].get("speed") for p in parsed if "usage" in p]
|
||||||
|
assert "standard" in speeds, parsed
|
||||||
|
|
||||||
|
|
||||||
|
def test_usage_speed_absent_when_anthropic_does_not_report(monkeypatch):
|
||||||
|
"""When the upstream stream omits ``usage.speed`` (pre-fast-mode
|
||||||
|
models / older snapshots), the Studio usage chunk must not invent
|
||||||
|
a value."""
|
||||||
|
_, lines = _capture(monkeypatch)
|
||||||
|
parsed = [
|
||||||
|
json.loads(l[len("data: ") :])
|
||||||
|
for l in lines
|
||||||
|
if l.startswith("data: ") and '"usage"' in l
|
||||||
|
]
|
||||||
|
for p in parsed:
|
||||||
|
usage = p.get("usage") or {}
|
||||||
|
assert "speed" not in usage, p
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@ from core.inference.pricing import (
|
||||||
ANTHROPIC_CACHE_5M_WRITE_MULT,
|
ANTHROPIC_CACHE_5M_WRITE_MULT,
|
||||||
ANTHROPIC_CACHE_1H_WRITE_MULT,
|
ANTHROPIC_CACHE_1H_WRITE_MULT,
|
||||||
ANTHROPIC_CACHE_READ_MULT,
|
ANTHROPIC_CACHE_READ_MULT,
|
||||||
|
ANTHROPIC_FAST_MODE_MULT,
|
||||||
ANTHROPIC_PRICING,
|
ANTHROPIC_PRICING,
|
||||||
OPENAI_CACHE_READ_MULT,
|
OPENAI_CACHE_READ_MULT,
|
||||||
OPENAI_CONTAINER_USD_PER_HOUR,
|
OPENAI_CONTAINER_USD_PER_HOUR,
|
||||||
|
|
@ -57,6 +58,65 @@ def test_anthropic_opus_4_7_input_and_output_math():
|
||||||
assert _isclose(out["total_usd"], 30.0)
|
assert _isclose(out["total_usd"], 30.0)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Anthropic fast-mode 6x multiplier (Opus 4.6 / 4.7 only) ─────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_anthropic_fast_mode_charges_6x_standard_opus():
|
||||||
|
"""Per https://platform.claude.com/docs/en/build-with-claude/fast-mode
|
||||||
|
fast-mode requests are billed at 6x the standard Opus rates across
|
||||||
|
the full context window. ``usage.speed == "fast"`` is the trigger."""
|
||||||
|
out = calculate_cost(
|
||||||
|
"anthropic",
|
||||||
|
"claude-opus-4-7",
|
||||||
|
{
|
||||||
|
"input_tokens": 1_000_000,
|
||||||
|
"output_tokens": 1_000_000,
|
||||||
|
"speed": "fast",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert _isclose(out["input_usd"], 5.0 * ANTHROPIC_FAST_MODE_MULT)
|
||||||
|
assert _isclose(out["output_usd"], 25.0 * ANTHROPIC_FAST_MODE_MULT)
|
||||||
|
assert _isclose(out["total_usd"], 30.0 * ANTHROPIC_FAST_MODE_MULT)
|
||||||
|
assert "(fast)" in out["model_priced"], out["model_priced"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_anthropic_fast_mode_does_not_affect_standard_speed():
|
||||||
|
"""``speed: "standard"`` (or missing) keeps the base rates."""
|
||||||
|
out_standard = calculate_cost(
|
||||||
|
"anthropic",
|
||||||
|
"claude-opus-4-7",
|
||||||
|
{
|
||||||
|
"input_tokens": 1_000_000,
|
||||||
|
"output_tokens": 1_000_000,
|
||||||
|
"speed": "standard",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
out_missing = calculate_cost(
|
||||||
|
"anthropic",
|
||||||
|
"claude-opus-4-7",
|
||||||
|
{"input_tokens": 1_000_000, "output_tokens": 1_000_000},
|
||||||
|
)
|
||||||
|
assert _isclose(out_standard["total_usd"], out_missing["total_usd"])
|
||||||
|
assert _isclose(out_standard["total_usd"], 30.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_anthropic_fast_mode_stacks_with_cache_read_multiplier():
|
||||||
|
"""Cache multipliers apply on top of fast-mode (per docs)."""
|
||||||
|
base = ANTHROPIC_PRICING["claude-opus-4-7"]["input_per_mtok"]
|
||||||
|
out = calculate_cost(
|
||||||
|
"anthropic",
|
||||||
|
"claude-opus-4-7",
|
||||||
|
{
|
||||||
|
"input_tokens": 0,
|
||||||
|
"output_tokens": 0,
|
||||||
|
"cache_read_input_tokens": 1_000_000,
|
||||||
|
"speed": "fast",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expected = base * ANTHROPIC_FAST_MODE_MULT * ANTHROPIC_CACHE_READ_MULT
|
||||||
|
assert _isclose(out["cache_read_usd"], expected)
|
||||||
|
|
||||||
|
|
||||||
# ── Anthropic cache write 5m + read multipliers ──────────────────────
|
# ── Anthropic cache write 5m + read multipliers ──────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -412,6 +472,7 @@ def test_snapshot_contains_provider_buckets_and_multipliers():
|
||||||
assert a["cache_5m_write_mult"] == ANTHROPIC_CACHE_5M_WRITE_MULT
|
assert a["cache_5m_write_mult"] == ANTHROPIC_CACHE_5M_WRITE_MULT
|
||||||
assert a["cache_1h_write_mult"] == ANTHROPIC_CACHE_1H_WRITE_MULT
|
assert a["cache_1h_write_mult"] == ANTHROPIC_CACHE_1H_WRITE_MULT
|
||||||
assert a["cache_read_mult"] == ANTHROPIC_CACHE_READ_MULT
|
assert a["cache_read_mult"] == ANTHROPIC_CACHE_READ_MULT
|
||||||
|
assert a["fast_mode_mult"] == ANTHROPIC_FAST_MODE_MULT
|
||||||
assert "web_search_usd_per_1k" in a
|
assert "web_search_usd_per_1k" in a
|
||||||
assert "code_execution_usd_per_hour" in a
|
assert "code_execution_usd_per_hour" in a
|
||||||
assert "models" in o and "gpt-5.5" in o["models"]
|
assert "models" in o and "gpt-5.5" in o["models"]
|
||||||
|
|
|
||||||
|
|
@ -341,12 +341,19 @@ function collectImageParts(
|
||||||
return parts;
|
return parts;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sentinel emitted by the backend at the end of an assistant turn that
|
// Out-of-band refusal flag stamped onto assistant message metadata by the
|
||||||
// Anthropic ended with stop_reason="refusal". We drop the entire turn
|
// adapter when the backend emits an `anthropic_refusal` _toolEvent. We
|
||||||
// from the next request body because Anthropic's guidance says leaving
|
// drop the entire turn from the next request body because Anthropic's
|
||||||
// the refused output in context causes the next call to keep refusing.
|
// guidance says leaving the refused output in context causes the next
|
||||||
// Keep in sync with studio/backend/core/inference/external_provider.py.
|
// call to keep refusing. Using metadata (not text) means assistant
|
||||||
const ANTHROPIC_REFUSAL_SENTINEL = "<!--studio:anthropic-refusal-->";
|
// content can never spoof a context reset.
|
||||||
|
function isAnthropicRefusalMessage(message: RunMessage): boolean {
|
||||||
|
if (message.role !== "assistant") return false;
|
||||||
|
const metadata = (message as { metadata?: unknown }).metadata as
|
||||||
|
| { custom?: Record<string, unknown> }
|
||||||
|
| undefined;
|
||||||
|
return metadata?.custom?.anthropicRefusal === true;
|
||||||
|
}
|
||||||
|
|
||||||
function toOpenAIMessage(message: RunMessage): {
|
function toOpenAIMessage(message: RunMessage): {
|
||||||
role: "system" | "user" | "assistant";
|
role: "system" | "user" | "assistant";
|
||||||
|
|
@ -368,7 +375,7 @@ function toOpenAIMessage(message: RunMessage): {
|
||||||
/data:audio\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g,
|
/data:audio\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g,
|
||||||
"[audio]",
|
"[audio]",
|
||||||
);
|
);
|
||||||
if (textContent.includes(ANTHROPIC_REFUSAL_SENTINEL)) {
|
if (isAnthropicRefusalMessage(message)) {
|
||||||
// Drop refused assistant turns entirely so the next request does
|
// Drop refused assistant turns entirely so the next request does
|
||||||
// not re-trigger the same safety classifier. The user-visible
|
// not re-trigger the same safety classifier. The user-visible
|
||||||
// notice stays in the rendered transcript; only the outbound
|
// notice stays in the rendered transcript; only the outbound
|
||||||
|
|
@ -938,14 +945,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
||||||
// guidance is explicit that leaving the offending user prompt in
|
// guidance is explicit that leaving the offending user prompt in
|
||||||
// context causes the next request to re-trigger the classifier;
|
// context causes the next request to re-trigger the classifier;
|
||||||
// returning null on just the assistant side was not enough.
|
// returning null on just the assistant side was not enough.
|
||||||
|
// The refusal flag rides assistant metadata.custom.anthropicRefusal
|
||||||
|
// (set out-of-band from the backend _toolEvent) so assistant text
|
||||||
|
// can never spoof a context reset.
|
||||||
const survivingMessages: RunMessage[] = [];
|
const survivingMessages: RunMessage[] = [];
|
||||||
for (const message of messages) {
|
for (const message of messages) {
|
||||||
if (
|
if (isAnthropicRefusalMessage(message)) {
|
||||||
message.role === "assistant" &&
|
|
||||||
collectTextParts(message)
|
|
||||||
.join("\n")
|
|
||||||
.includes(ANTHROPIC_REFUSAL_SENTINEL)
|
|
||||||
) {
|
|
||||||
const last = survivingMessages.at(-1);
|
const last = survivingMessages.at(-1);
|
||||||
if (last && last.role === "user") {
|
if (last && last.role === "user") {
|
||||||
survivingMessages.pop();
|
survivingMessages.pop();
|
||||||
|
|
@ -1025,8 +1030,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const imageBase64 = findLatestUserImageBase64(messages);
|
// Scan the post-prune history so a refused user turn with an
|
||||||
const audioBase64 = findLatestUserAudioBase64(messages);
|
// image/audio attachment does not gate / mis-attribute the next
|
||||||
|
// non-refused turn (the refused pair is pruned from the request
|
||||||
|
// body above).
|
||||||
|
const imageBase64 = findLatestUserImageBase64(survivingMessages);
|
||||||
|
const audioBase64 = findLatestUserAudioBase64(survivingMessages);
|
||||||
|
|
||||||
// Block when ANY image is in the outbound payload (current or
|
// Block when ANY image is in the outbound payload (current or
|
||||||
// prior turns) and the loaded model can't process images. Keeps
|
// prior turns) and the loaded model can't process images. Keeps
|
||||||
|
|
@ -1062,7 +1071,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
||||||
if (audioBase64) {
|
if (audioBase64) {
|
||||||
const audioName = runtime.pendingAudioName;
|
const audioName = runtime.pendingAudioName;
|
||||||
if (audioName) {
|
if (audioName) {
|
||||||
const lastUserMsg = [...messages]
|
const lastUserMsg = [...survivingMessages]
|
||||||
.reverse()
|
.reverse()
|
||||||
.find((m) => m.role === "user");
|
.find((m) => m.role === "user");
|
||||||
if (lastUserMsg) sentAudioNames.set(lastUserMsg.id, audioName);
|
if (lastUserMsg) sentAudioNames.set(lastUserMsg.id, audioName);
|
||||||
|
|
@ -1173,6 +1182,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
||||||
// Tool call content parts — accumulated and yielded cumulatively.
|
// Tool call content parts — accumulated and yielded cumulatively.
|
||||||
// result is set directly on the tool-call part when tool_end arrives.
|
// result is set directly on the tool-call part when tool_end arrives.
|
||||||
const toolCallParts: ToolCallMessagePart[] = [];
|
const toolCallParts: ToolCallMessagePart[] = [];
|
||||||
|
// Latched when the backend emits an `anthropic_refusal` tool event
|
||||||
|
// on Anthropic stop_reason="refusal". Stamped onto the final
|
||||||
|
// assistant message metadata as `custom.anthropicRefusal` so the
|
||||||
|
// history-prune logic above can drop the refused pair on the next
|
||||||
|
// request without relying on text-content sentinels.
|
||||||
|
let anthropicRefusalSeen = false;
|
||||||
let serverMetadata: {
|
let serverMetadata: {
|
||||||
usage?: ServerUsage;
|
usage?: ServerUsage;
|
||||||
timings?: ServerTimings;
|
timings?: ServerTimings;
|
||||||
|
|
@ -1645,6 +1660,14 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (toolEvent.type === "anthropic_refusal") {
|
||||||
|
// Backend signalled stop_reason="refusal" out-of-band.
|
||||||
|
// Latch and stamp onto the final message metadata so
|
||||||
|
// the two-pass history pruner can drop the refused
|
||||||
|
// pair on the next request.
|
||||||
|
anthropicRefusalSeen = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (toolEvent.type === "tool_start") {
|
if (toolEvent.type === "tool_start") {
|
||||||
const id =
|
const id =
|
||||||
(toolEvent.tool_call_id as string) ||
|
(toolEvent.tool_call_id as string) ||
|
||||||
|
|
@ -1965,6 +1988,10 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
||||||
timing: finalTiming,
|
timing: finalTiming,
|
||||||
custom: {
|
custom: {
|
||||||
reasoningDuration,
|
reasoningDuration,
|
||||||
|
// Persisted refusal flag; drives the two-pass history
|
||||||
|
// pruner that drops the refused assistant + user pair
|
||||||
|
// on the next request.
|
||||||
|
anthropicRefusal: anthropicRefusalSeen || undefined,
|
||||||
serverTimings: meta?.timings ?? undefined,
|
serverTimings: meta?.timings ?? undefined,
|
||||||
contextUsage: meta?.usage
|
contextUsage: meta?.usage
|
||||||
? {
|
? {
|
||||||
|
|
|
||||||
|
|
@ -155,8 +155,12 @@ export function providerSupportsFastMode(
|
||||||
): boolean {
|
): boolean {
|
||||||
if (providerType !== "anthropic") return false;
|
if (providerType !== "anthropic") return false;
|
||||||
if (!modelId) return false;
|
if (!modelId) return false;
|
||||||
return ANTHROPIC_FAST_MODE_MODEL_PREFIXES.some((prefix) =>
|
// Require the prefix to terminate at a family boundary (end of
|
||||||
modelId.startsWith(prefix),
|
// string or "-" before a dated snapshot) so the check does not
|
||||||
|
// match unsupported IDs that merely share a prefix, e.g.
|
||||||
|
// "claude-opus-4-70" / "claude-opus-4-7b".
|
||||||
|
return ANTHROPIC_FAST_MODE_MODEL_PREFIXES.some(
|
||||||
|
(prefix) => modelId === prefix || modelId.startsWith(`${prefix}-`),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue