Studio: out-of-band refusal signal + fast-mode prefix/usage/pricing fixes
The text sentinel for the Anthropic refusal drop signal was spoofable: any assistant message containing the literal <!--studio:anthropic-refusal--> would prune the prior user + assistant pair on the next request. Move the signal onto a separate _toolEvent chunk that the chat adapter latches into assistant.metadata.custom.anthropicRefusal; assistant text can no longer control the pruner. Tighten the fast-mode model gate (backend + frontend) to require a "-" family boundary so claude-opus-4-70 / claude-opus-4-7b style IDs do not get speed: "fast" on a naive startswith match. Use survivingMessages for the image / audio attachment scan so a refused user turn does not gate or mis-attribute the next non-refused turn. Propagate Anthropic usage.speed onto the OpenAI-style usage chunk and apply the documented 6x fast-mode multiplier in the cost calculator (stacks with prompt-cache multipliers per the docs); expose the new multiplier on the pricing snapshot for the UI tooltip. Tests cover the tool-event chunk shape, the prefix-collision rejects, usage.speed propagation, the 6x pricing math, and that the visible refusal text carries no embedded sentinel.
This commit is contained in:
Daniel Han
parent
41531cdbe2
commit
4f1afdb535
7 changed files with 302 additions and 48 deletions
|
|
@ -191,7 +191,15 @@ def _anthropic_supports_compaction(model: str) -> bool:
|
|||
|
||||
|
||||
def _anthropic_supports_fast_mode(model: str) -> bool:
|
||||
return model.startswith(_ANTHROPIC_FAST_MODE_PREFIXES)
|
||||
# Require the prefix to terminate at a family boundary (end of
|
||||
# string or a "-" separator before the date snapshot) so the
|
||||
# check does not light up on hypothetical IDs like
|
||||
# "claude-opus-4-70" / "claude-opus-4-7b" that merely share a
|
||||
# prefix with the supported families.
|
||||
return any(
|
||||
model == p or model.startswith(f"{p}-")
|
||||
for p in _ANTHROPIC_FAST_MODE_PREFIXES
|
||||
)
|
||||
|
||||
|
||||
class _MistralThinkingSpec(NamedTuple):
|
||||
|
|
@ -2466,20 +2474,27 @@ class ExternalProviderClient:
|
|||
"Anthropic refusal stop_reason (model=%s)",
|
||||
model,
|
||||
)
|
||||
# Trailing HTML comment is a stable
|
||||
# sentinel the chat-adapter matches
|
||||
# to drop this assistant turn from
|
||||
# the next request's outbound
|
||||
# history. Anthropic's docs say
|
||||
# the refused turn must be removed
|
||||
# or updated before continuing or
|
||||
# the next call will keep refusing.
|
||||
# User-facing notice so the chat bubble
|
||||
# is not silently empty after the
|
||||
# safety classifier truncates the
|
||||
# response. The out-of-band drop
|
||||
# signal rides a separate _toolEvent
|
||||
# below so assistant text can never
|
||||
# spoof a context reset by including
|
||||
# a literal marker. Anthropic's docs
|
||||
# say the refused turn must be
|
||||
# removed or updated before
|
||||
# continuing or the next call will
|
||||
# keep refusing.
|
||||
# https://platform.claude.com/docs/en/test-and-evaluate/strengthen-guardrails/handle-streaming-refusals
|
||||
yield _content_chunk(
|
||||
"\n\n_The response was stopped by "
|
||||
"Anthropic's safety classifier. Edit "
|
||||
"or remove the previous turn and try "
|
||||
"again._"
|
||||
"\n<!--studio:anthropic-refusal-->"
|
||||
)
|
||||
yield _emit_tool_event(
|
||||
{"type": "anthropic_refusal"}
|
||||
)
|
||||
if mapped is not None:
|
||||
chunk = {
|
||||
|
|
@ -4010,6 +4025,15 @@ def _build_usage_chunk(
|
|||
"cache_creation_input_tokens": cache_creation,
|
||||
"cache_read_input_tokens": cache_read,
|
||||
}
|
||||
# Anthropic fast-mode responses include `usage.speed` so callers
|
||||
# can verify whether a premium fast-mode request actually ran
|
||||
# fast (the API silently falls back to "standard" when the
|
||||
# beta is unsupported or rate-limited). Surface it on the
|
||||
# OpenAI-style chunk so the pricing/cost ledger can apply the
|
||||
# 6x multiplier without re-derivation.
|
||||
speed = last_usage.get("speed")
|
||||
if speed in ("fast", "standard"):
|
||||
usage_block["speed"] = speed
|
||||
else:
|
||||
prompt_tokens = last_usage.get("input_tokens") or 0
|
||||
cached = 0
|
||||
|
|
|
|||
|
|
@ -105,6 +105,10 @@ OPENAI_PRICING: dict[str, dict[str, float]] = {
|
|||
ANTHROPIC_CACHE_5M_WRITE_MULT = 1.25
|
||||
ANTHROPIC_CACHE_1H_WRITE_MULT = 2.0
|
||||
ANTHROPIC_CACHE_READ_MULT = 0.1
|
||||
# Anthropic fast-mode beta (Opus 4.6 / 4.7 only): 6x standard rates on
|
||||
# both input and output across the full context window.
|
||||
# https://platform.claude.com/docs/en/build-with-claude/fast-mode#pricing
|
||||
ANTHROPIC_FAST_MODE_MULT = 6.0
|
||||
|
||||
# OpenAI: cache reads are 0.1x base input, cache writes are not billed
|
||||
# separately (the first prefix-write request just pays normal input).
|
||||
|
|
@ -235,6 +239,17 @@ def calculate_cost(
|
|||
base = prices["input_per_mtok"]
|
||||
out_per = prices["output_per_mtok"]
|
||||
|
||||
# Anthropic fast-mode beta: 6x standard on both input + output across
|
||||
# the full context window. Prompt-cache multipliers stack on top of
|
||||
# the fast-mode base per the docs, so applying the multiplier once
|
||||
# to (base, out_per) propagates correctly into the cache_*_usd
|
||||
# buckets computed below.
|
||||
if provider == "anthropic" and usage.get("speed") == "fast":
|
||||
base *= ANTHROPIC_FAST_MODE_MULT
|
||||
out_per *= ANTHROPIC_FAST_MODE_MULT
|
||||
if out["model_priced"]:
|
||||
out["model_priced"] = f"{out['model_priced']} (fast)"
|
||||
|
||||
out["input_usd"] = (input_tokens / 1_000_000.0) * base
|
||||
out["output_usd"] = (output_tokens / 1_000_000.0) * out_per
|
||||
|
||||
|
|
@ -315,6 +330,7 @@ def pricing_snapshot() -> dict[str, Any]:
|
|||
"cache_5m_write_mult": ANTHROPIC_CACHE_5M_WRITE_MULT,
|
||||
"cache_1h_write_mult": ANTHROPIC_CACHE_1H_WRITE_MULT,
|
||||
"cache_read_mult": ANTHROPIC_CACHE_READ_MULT,
|
||||
"fast_mode_mult": ANTHROPIC_FAST_MODE_MULT,
|
||||
"web_search_usd_per_1k": ANTHROPIC_WEB_SEARCH_USD_PER_1K,
|
||||
"code_execution_usd_per_hour": ANTHROPIC_CODE_EXEC_USD_PER_HOUR,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -160,14 +160,20 @@ def test_refusal_emits_user_facing_notice_and_content_filter_finish(monkeypatch)
|
|||
assert "Hello." in body, body
|
||||
|
||||
|
||||
def test_refusal_emits_hidden_sentinel_for_chat_adapter_drop(monkeypatch):
|
||||
"""Refused turns embed a hidden sentinel the chat-adapter matches on.
|
||||
def test_refusal_emits_tool_event_for_chat_adapter_drop(monkeypatch):
|
||||
"""Refused turns emit an out-of-band `_toolEvent` the chat-adapter
|
||||
latches into assistant `metadata.custom.anthropicRefusal`.
|
||||
|
||||
The frontend drops any assistant message containing this sentinel
|
||||
The frontend drops any assistant message with that metadata flag
|
||||
from the next request's outbound history, per Anthropic's docs:
|
||||
leaving the refused output in context causes the next call to keep
|
||||
refusing.
|
||||
refusing. Using a tool event (not a text sentinel) means assistant
|
||||
content can never spoof a context reset.
|
||||
"""
|
||||
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
||||
body = "\n".join(lines)
|
||||
assert "studio:anthropic-refusal" in body, body
|
||||
assert '"_toolEvent": {"type": "anthropic_refusal"}' in body, body
|
||||
# The visible refusal notice must NOT carry a text sentinel: an
|
||||
# assistant message that echoes that string would otherwise spoof
|
||||
# a context reset on the next request.
|
||||
assert "studio:anthropic-refusal" not in body, body
|
||||
|
|
|
|||
|
|
@ -23,14 +23,16 @@ This file fills in the remaining behaviour cliffs:
|
|||
duplicates and no truncation.
|
||||
* Idempotence: setting ``fast_mode=True`` twice via the same body still
|
||||
results in one beta-header entry.
|
||||
* Streaming refusal sentinel: emitted exactly once, with the exact
|
||||
``<!--studio:anthropic-refusal-->`` token the frontend matches on,
|
||||
and the notice always precedes the finish_reason chunk so a UI
|
||||
* Streaming refusal signal: a single out-of-band ``_toolEvent`` carrying
|
||||
``{"type": "anthropic_refusal"}`` rides alongside the visible refusal
|
||||
notice. The frontend latches the tool event into assistant
|
||||
metadata.custom.anthropicRefusal; assistant text never controls the
|
||||
pruner. The notice always precedes the finish_reason chunk so a UI
|
||||
reading the SSE in order paints text before flipping to
|
||||
``content_filter``.
|
||||
* Refusal on a non-Opus model: refusal handling is provider-side, not
|
||||
model-gated, so a refusal mid-stream on Sonnet must still surface the
|
||||
notice + sentinel.
|
||||
notice + tool event.
|
||||
* Non-destruction: when ``fast_mode`` is ``None``, the outbound body
|
||||
and headers must be byte-identical to the version that omits the
|
||||
argument entirely. This guarantees the upgrade path is non-breaking
|
||||
|
|
@ -328,30 +330,41 @@ def test_refusal_notice_appears_before_content_filter_chunk(monkeypatch):
|
|||
assert notice_idx < filter_idx, (notice_idx, filter_idx, lines)
|
||||
|
||||
|
||||
def test_refusal_sentinel_emitted_exactly_once(monkeypatch):
|
||||
"""A single refusal must emit the chat-adapter drop sentinel one time.
|
||||
def test_refusal_tool_event_emitted_exactly_once(monkeypatch):
|
||||
"""A single refusal must emit the chat-adapter drop signal one time.
|
||||
|
||||
The frontend's ``toOpenAIMessage`` uses ``includes`` for the match,
|
||||
so duplicates wouldn't cause false drops -- but emitting it twice
|
||||
would still inflate the bubble and break the UX guarantee that the
|
||||
sentinel is an invisible HTML comment.
|
||||
The frontend latches the tool event into assistant metadata and
|
||||
uses it to drop the refused pair from the next request. Emitting
|
||||
twice would still be metadata-idempotent but indicates a backend
|
||||
bug, so pin the count.
|
||||
"""
|
||||
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
||||
body = "\n".join(lines)
|
||||
count = body.count("<!--studio:anthropic-refusal-->")
|
||||
count = body.count('"_toolEvent": {"type": "anthropic_refusal"}')
|
||||
assert count == 1, (count, body)
|
||||
|
||||
|
||||
def test_refusal_text_carries_no_html_sentinel(monkeypatch):
|
||||
"""Belt-and-braces: ensure the assistant-visible refusal text does
|
||||
not embed any ``studio:anthropic-refusal`` marker. The drop signal
|
||||
must ride the out-of-band ``_toolEvent`` channel only -- otherwise
|
||||
an assistant message that echoes the literal marker would spoof a
|
||||
context reset on the next request."""
|
||||
_, lines = _capture(monkeypatch, sse = _refusal_sse())
|
||||
body = "\n".join(lines)
|
||||
assert "studio:anthropic-refusal" not in body, body
|
||||
|
||||
|
||||
def test_refusal_handling_works_on_sonnet_model(monkeypatch):
|
||||
"""Refusal handling is provider-side, not gated on a fast-mode-capable
|
||||
model. If Anthropic's classifier refuses a Sonnet stream it must
|
||||
surface the same notice + sentinel + content_filter mapping."""
|
||||
surface the same notice + tool event + content_filter mapping."""
|
||||
_, lines = _capture(
|
||||
monkeypatch, sse = _refusal_sse("claude-sonnet-4-6"), model = "claude-sonnet-4-6"
|
||||
)
|
||||
body = "\n".join(lines)
|
||||
assert "stopped by Anthropic's safety classifier" in body, body
|
||||
assert "<!--studio:anthropic-refusal-->" in body, body
|
||||
assert '"_toolEvent": {"type": "anthropic_refusal"}' in body, body
|
||||
assert '"finish_reason": "content_filter"' in body, body
|
||||
|
||||
|
||||
|
|
@ -369,7 +382,8 @@ def test_refusal_preserves_partial_assistant_text(monkeypatch):
|
|||
def test_refusal_chunk_is_proper_openai_delta_shape(monkeypatch):
|
||||
"""The notice rides a normal ``choices[0].delta.content`` chunk, not
|
||||
a finish_reason chunk, so OpenAI-spec clients (Aurora, OpenAI SDK)
|
||||
treat it as ordinary streamed text."""
|
||||
treat it as ordinary streamed text. The drop signal arrives on a
|
||||
separate `_toolEvent` chunk (verified below)."""
|
||||
_, lines = _capture(monkeypatch, sse = _refusal_sse(), model = "claude-opus-4-7")
|
||||
# Find the chunk that carries the refusal text.
|
||||
notice_chunk = None
|
||||
|
|
@ -383,7 +397,27 @@ def test_refusal_chunk_is_proper_openai_delta_shape(monkeypatch):
|
|||
# Must NOT carry a finish_reason itself -- that comes on the next
|
||||
# chunk.
|
||||
assert choice.get("finish_reason") in (None,), notice_chunk
|
||||
assert "<!--studio:anthropic-refusal-->" in choice["delta"]["content"]
|
||||
# Refusal text is plain-spoken; no embedded sentinel.
|
||||
assert "studio:anthropic-refusal" not in choice["delta"]["content"]
|
||||
|
||||
|
||||
def test_refusal_tool_event_chunk_shape(monkeypatch):
|
||||
"""The out-of-band drop signal rides a separate chunk shaped like a
|
||||
Studio `_toolEvent` envelope (choices=[{index:0, delta:{},
|
||||
finish_reason:null}] + `_toolEvent`). The frontend latches on
|
||||
`_toolEvent.type == "anthropic_refusal"` and stamps the assistant
|
||||
message metadata; assistant text never controls the prune."""
|
||||
_, lines = _capture(monkeypatch, sse = _refusal_sse(), model = "claude-opus-4-7")
|
||||
refusal_chunk = None
|
||||
for line in lines:
|
||||
if line.startswith("data: ") and "anthropic_refusal" in line:
|
||||
refusal_chunk = json.loads(line[len("data: ") :])
|
||||
break
|
||||
assert refusal_chunk is not None, lines
|
||||
assert refusal_chunk["_toolEvent"] == {"type": "anthropic_refusal"}, refusal_chunk
|
||||
choice = refusal_chunk["choices"][0]
|
||||
assert choice["delta"] == {}, refusal_chunk
|
||||
assert choice["finish_reason"] is None, refusal_chunk
|
||||
|
||||
|
||||
# ──────────────────────────── future-proofing ────────────────────────────
|
||||
|
|
@ -415,3 +449,85 @@ def test_fast_mode_dropped_on_opus_4_5_dated_snapshot(monkeypatch):
|
|||
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-5-2025-08-01")
|
||||
assert "speed" not in cap["body"], cap["body"]
|
||||
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||
|
||||
|
||||
def test_fast_mode_rejects_prefix_collision_4_70(monkeypatch):
|
||||
"""The family gate must require a "-" boundary after the supported
|
||||
prefix so hypothetical IDs like ``claude-opus-4-70`` or
|
||||
``claude-opus-4-7b`` do not get fast-mode on a naive
|
||||
``startswith`` match."""
|
||||
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-70")
|
||||
assert "speed" not in cap["body"], cap["body"]
|
||||
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||
|
||||
|
||||
def test_fast_mode_rejects_prefix_collision_4_7b(monkeypatch):
|
||||
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-7b")
|
||||
assert "speed" not in cap["body"], cap["body"]
|
||||
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||
|
||||
|
||||
def test_fast_mode_rejects_prefix_collision_4_6_extra(monkeypatch):
|
||||
cap, _ = _capture(monkeypatch, fast_mode = True, model = "claude-opus-4-60")
|
||||
assert "speed" not in cap["body"], cap["body"]
|
||||
assert "fast-mode-2026-02-01" not in cap["headers"].get("anthropic-beta", "")
|
||||
|
||||
|
||||
# ──────────────────────────── usage.speed propagation ────────────────────────────
|
||||
def _fast_speed_sse(model: str = "claude-opus-4-7", speed: str = "fast") -> bytes:
|
||||
return (
|
||||
b'event: message_start\ndata: {"type":"message_start","message":'
|
||||
b'{"id":"m1","content":[],"model":"' + model.encode() + b'",'
|
||||
b'"role":"assistant","stop_reason":null,"usage":'
|
||||
b'{"input_tokens":4,"output_tokens":1}}}\n\n'
|
||||
b'event: content_block_start\ndata: {"type":"content_block_start",'
|
||||
b'"index":0,"content_block":{"type":"text","text":""}}\n\n'
|
||||
b'event: content_block_delta\ndata: {"type":"content_block_delta",'
|
||||
b'"index":0,"delta":{"type":"text_delta","text":"hi"}}\n\n'
|
||||
b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n'
|
||||
b'event: message_delta\ndata: {"type":"message_delta",'
|
||||
b'"delta":{"stop_reason":"end_turn"},'
|
||||
b'"usage":{"output_tokens":5,"speed":"' + speed.encode() + b'"}}\n\n'
|
||||
b'event: message_stop\ndata: {"type":"message_stop"}\n\n'
|
||||
)
|
||||
|
||||
|
||||
def test_usage_speed_propagates_to_final_usage_chunk_fast(monkeypatch):
|
||||
"""When Anthropic returns ``usage.speed == "fast"`` the Studio
|
||||
OpenAI-style usage chunk must carry that field so the cost ledger
|
||||
can apply the 6x multiplier and clients can verify a fast-mode
|
||||
request actually ran fast."""
|
||||
_, lines = _capture(monkeypatch, sse = _fast_speed_sse(speed = "fast"))
|
||||
usage_lines = [
|
||||
l for l in lines if l.startswith("data: ") and '"usage"' in l
|
||||
]
|
||||
assert usage_lines, lines
|
||||
parsed = [json.loads(l[len("data: ") :]) for l in usage_lines]
|
||||
speeds = [p["usage"].get("speed") for p in parsed if "usage" in p]
|
||||
assert "fast" in speeds, parsed
|
||||
|
||||
|
||||
def test_usage_speed_propagates_to_final_usage_chunk_standard(monkeypatch):
|
||||
_, lines = _capture(monkeypatch, sse = _fast_speed_sse(speed = "standard"))
|
||||
parsed = [
|
||||
json.loads(l[len("data: ") :])
|
||||
for l in lines
|
||||
if l.startswith("data: ") and '"usage"' in l
|
||||
]
|
||||
speeds = [p["usage"].get("speed") for p in parsed if "usage" in p]
|
||||
assert "standard" in speeds, parsed
|
||||
|
||||
|
||||
def test_usage_speed_absent_when_anthropic_does_not_report(monkeypatch):
|
||||
"""When the upstream stream omits ``usage.speed`` (pre-fast-mode
|
||||
models / older snapshots), the Studio usage chunk must not invent
|
||||
a value."""
|
||||
_, lines = _capture(monkeypatch)
|
||||
parsed = [
|
||||
json.loads(l[len("data: ") :])
|
||||
for l in lines
|
||||
if l.startswith("data: ") and '"usage"' in l
|
||||
]
|
||||
for p in parsed:
|
||||
usage = p.get("usage") or {}
|
||||
assert "speed" not in usage, p
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from core.inference.pricing import (
|
|||
ANTHROPIC_CACHE_5M_WRITE_MULT,
|
||||
ANTHROPIC_CACHE_1H_WRITE_MULT,
|
||||
ANTHROPIC_CACHE_READ_MULT,
|
||||
ANTHROPIC_FAST_MODE_MULT,
|
||||
ANTHROPIC_PRICING,
|
||||
OPENAI_CACHE_READ_MULT,
|
||||
OPENAI_CONTAINER_USD_PER_HOUR,
|
||||
|
|
@ -57,6 +58,65 @@ def test_anthropic_opus_4_7_input_and_output_math():
|
|||
assert _isclose(out["total_usd"], 30.0)
|
||||
|
||||
|
||||
# ── Anthropic fast-mode 6x multiplier (Opus 4.6 / 4.7 only) ─────────
|
||||
|
||||
|
||||
def test_anthropic_fast_mode_charges_6x_standard_opus():
|
||||
"""Per https://platform.claude.com/docs/en/build-with-claude/fast-mode
|
||||
fast-mode requests are billed at 6x the standard Opus rates across
|
||||
the full context window. ``usage.speed == "fast"`` is the trigger."""
|
||||
out = calculate_cost(
|
||||
"anthropic",
|
||||
"claude-opus-4-7",
|
||||
{
|
||||
"input_tokens": 1_000_000,
|
||||
"output_tokens": 1_000_000,
|
||||
"speed": "fast",
|
||||
},
|
||||
)
|
||||
assert _isclose(out["input_usd"], 5.0 * ANTHROPIC_FAST_MODE_MULT)
|
||||
assert _isclose(out["output_usd"], 25.0 * ANTHROPIC_FAST_MODE_MULT)
|
||||
assert _isclose(out["total_usd"], 30.0 * ANTHROPIC_FAST_MODE_MULT)
|
||||
assert "(fast)" in out["model_priced"], out["model_priced"]
|
||||
|
||||
|
||||
def test_anthropic_fast_mode_does_not_affect_standard_speed():
|
||||
"""``speed: "standard"`` (or missing) keeps the base rates."""
|
||||
out_standard = calculate_cost(
|
||||
"anthropic",
|
||||
"claude-opus-4-7",
|
||||
{
|
||||
"input_tokens": 1_000_000,
|
||||
"output_tokens": 1_000_000,
|
||||
"speed": "standard",
|
||||
},
|
||||
)
|
||||
out_missing = calculate_cost(
|
||||
"anthropic",
|
||||
"claude-opus-4-7",
|
||||
{"input_tokens": 1_000_000, "output_tokens": 1_000_000},
|
||||
)
|
||||
assert _isclose(out_standard["total_usd"], out_missing["total_usd"])
|
||||
assert _isclose(out_standard["total_usd"], 30.0)
|
||||
|
||||
|
||||
def test_anthropic_fast_mode_stacks_with_cache_read_multiplier():
|
||||
"""Cache multipliers apply on top of fast-mode (per docs)."""
|
||||
base = ANTHROPIC_PRICING["claude-opus-4-7"]["input_per_mtok"]
|
||||
out = calculate_cost(
|
||||
"anthropic",
|
||||
"claude-opus-4-7",
|
||||
{
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"cache_read_input_tokens": 1_000_000,
|
||||
"speed": "fast",
|
||||
},
|
||||
)
|
||||
expected = base * ANTHROPIC_FAST_MODE_MULT * ANTHROPIC_CACHE_READ_MULT
|
||||
assert _isclose(out["cache_read_usd"], expected)
|
||||
|
||||
|
||||
# ── Anthropic cache write 5m + read multipliers ──────────────────────
|
||||
|
||||
|
||||
|
|
@ -412,6 +472,7 @@ def test_snapshot_contains_provider_buckets_and_multipliers():
|
|||
assert a["cache_5m_write_mult"] == ANTHROPIC_CACHE_5M_WRITE_MULT
|
||||
assert a["cache_1h_write_mult"] == ANTHROPIC_CACHE_1H_WRITE_MULT
|
||||
assert a["cache_read_mult"] == ANTHROPIC_CACHE_READ_MULT
|
||||
assert a["fast_mode_mult"] == ANTHROPIC_FAST_MODE_MULT
|
||||
assert "web_search_usd_per_1k" in a
|
||||
assert "code_execution_usd_per_hour" in a
|
||||
assert "models" in o and "gpt-5.5" in o["models"]
|
||||
|
|
|
|||
|
|
@ -341,12 +341,19 @@ function collectImageParts(
|
|||
return parts;
|
||||
}
|
||||
|
||||
// Sentinel emitted by the backend at the end of an assistant turn that
|
||||
// Anthropic ended with stop_reason="refusal". We drop the entire turn
|
||||
// from the next request body because Anthropic's guidance says leaving
|
||||
// the refused output in context causes the next call to keep refusing.
|
||||
// Keep in sync with studio/backend/core/inference/external_provider.py.
|
||||
const ANTHROPIC_REFUSAL_SENTINEL = "<!--studio:anthropic-refusal-->";
|
||||
// Out-of-band refusal flag stamped onto assistant message metadata by the
|
||||
// adapter when the backend emits an `anthropic_refusal` _toolEvent. We
|
||||
// drop the entire turn from the next request body because Anthropic's
|
||||
// guidance says leaving the refused output in context causes the next
|
||||
// call to keep refusing. Using metadata (not text) means assistant
|
||||
// content can never spoof a context reset.
|
||||
function isAnthropicRefusalMessage(message: RunMessage): boolean {
|
||||
if (message.role !== "assistant") return false;
|
||||
const metadata = (message as { metadata?: unknown }).metadata as
|
||||
| { custom?: Record<string, unknown> }
|
||||
| undefined;
|
||||
return metadata?.custom?.anthropicRefusal === true;
|
||||
}
|
||||
|
||||
function toOpenAIMessage(message: RunMessage): {
|
||||
role: "system" | "user" | "assistant";
|
||||
|
|
@ -368,7 +375,7 @@ function toOpenAIMessage(message: RunMessage): {
|
|||
/data:audio\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/g,
|
||||
"[audio]",
|
||||
);
|
||||
if (textContent.includes(ANTHROPIC_REFUSAL_SENTINEL)) {
|
||||
if (isAnthropicRefusalMessage(message)) {
|
||||
// Drop refused assistant turns entirely so the next request does
|
||||
// not re-trigger the same safety classifier. The user-visible
|
||||
// notice stays in the rendered transcript; only the outbound
|
||||
|
|
@ -938,14 +945,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
// guidance is explicit that leaving the offending user prompt in
|
||||
// context causes the next request to re-trigger the classifier;
|
||||
// returning null on just the assistant side was not enough.
|
||||
// The refusal flag rides assistant metadata.custom.anthropicRefusal
|
||||
// (set out-of-band from the backend _toolEvent) so assistant text
|
||||
// can never spoof a context reset.
|
||||
const survivingMessages: RunMessage[] = [];
|
||||
for (const message of messages) {
|
||||
if (
|
||||
message.role === "assistant" &&
|
||||
collectTextParts(message)
|
||||
.join("\n")
|
||||
.includes(ANTHROPIC_REFUSAL_SENTINEL)
|
||||
) {
|
||||
if (isAnthropicRefusalMessage(message)) {
|
||||
const last = survivingMessages.at(-1);
|
||||
if (last && last.role === "user") {
|
||||
survivingMessages.pop();
|
||||
|
|
@ -1025,8 +1030,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
});
|
||||
}
|
||||
}
|
||||
const imageBase64 = findLatestUserImageBase64(messages);
|
||||
const audioBase64 = findLatestUserAudioBase64(messages);
|
||||
// Scan the post-prune history so a refused user turn with an
|
||||
// image/audio attachment does not gate / mis-attribute the next
|
||||
// non-refused turn (the refused pair is pruned from the request
|
||||
// body above).
|
||||
const imageBase64 = findLatestUserImageBase64(survivingMessages);
|
||||
const audioBase64 = findLatestUserAudioBase64(survivingMessages);
|
||||
|
||||
// Block when ANY image is in the outbound payload (current or
|
||||
// prior turns) and the loaded model can't process images. Keeps
|
||||
|
|
@ -1062,7 +1071,7 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
if (audioBase64) {
|
||||
const audioName = runtime.pendingAudioName;
|
||||
if (audioName) {
|
||||
const lastUserMsg = [...messages]
|
||||
const lastUserMsg = [...survivingMessages]
|
||||
.reverse()
|
||||
.find((m) => m.role === "user");
|
||||
if (lastUserMsg) sentAudioNames.set(lastUserMsg.id, audioName);
|
||||
|
|
@ -1173,6 +1182,12 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
// Tool call content parts — accumulated and yielded cumulatively.
|
||||
// result is set directly on the tool-call part when tool_end arrives.
|
||||
const toolCallParts: ToolCallMessagePart[] = [];
|
||||
// Latched when the backend emits an `anthropic_refusal` tool event
|
||||
// on Anthropic stop_reason="refusal". Stamped onto the final
|
||||
// assistant message metadata as `custom.anthropicRefusal` so the
|
||||
// history-prune logic above can drop the refused pair on the next
|
||||
// request without relying on text-content sentinels.
|
||||
let anthropicRefusalSeen = false;
|
||||
let serverMetadata: {
|
||||
usage?: ServerUsage;
|
||||
timings?: ServerTimings;
|
||||
|
|
@ -1645,6 +1660,14 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
}
|
||||
continue;
|
||||
}
|
||||
if (toolEvent.type === "anthropic_refusal") {
|
||||
// Backend signalled stop_reason="refusal" out-of-band.
|
||||
// Latch and stamp onto the final message metadata so
|
||||
// the two-pass history pruner can drop the refused
|
||||
// pair on the next request.
|
||||
anthropicRefusalSeen = true;
|
||||
continue;
|
||||
}
|
||||
if (toolEvent.type === "tool_start") {
|
||||
const id =
|
||||
(toolEvent.tool_call_id as string) ||
|
||||
|
|
@ -1965,6 +1988,10 @@ export function createOpenAIStreamAdapter(): ChatModelAdapter {
|
|||
timing: finalTiming,
|
||||
custom: {
|
||||
reasoningDuration,
|
||||
// Persisted refusal flag; drives the two-pass history
|
||||
// pruner that drops the refused assistant + user pair
|
||||
// on the next request.
|
||||
anthropicRefusal: anthropicRefusalSeen || undefined,
|
||||
serverTimings: meta?.timings ?? undefined,
|
||||
contextUsage: meta?.usage
|
||||
? {
|
||||
|
|
|
|||
|
|
@ -155,8 +155,12 @@ export function providerSupportsFastMode(
|
|||
): boolean {
|
||||
if (providerType !== "anthropic") return false;
|
||||
if (!modelId) return false;
|
||||
return ANTHROPIC_FAST_MODE_MODEL_PREFIXES.some((prefix) =>
|
||||
modelId.startsWith(prefix),
|
||||
// Require the prefix to terminate at a family boundary (end of
|
||||
// string or "-" before a dated snapshot) so the check does not
|
||||
// match unsupported IDs that merely share a prefix, e.g.
|
||||
// "claude-opus-4-70" / "claude-opus-4-7b".
|
||||
return ANTHROPIC_FAST_MODE_MODEL_PREFIXES.some(
|
||||
(prefix) => modelId === prefix || modelId.startsWith(`${prefix}-`),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue