From f399e3b9d1f5a5feee5eebcf41e13dc7fce5235e Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 06:03:27 -0700 Subject: [PATCH] Studio: per-model Anthropic server-side tool versions (#5679) * Studio: per-model Anthropic server-side tool versions Anthropic ships date-pinned tool versions per model family. Studio currently hard-codes `web_search_20250305`, `web_fetch_20250910`, and `code_execution_20250825` for every model, which means Opus 4.6/4.7, Sonnet 4.6 and the Opus/Sonnet 4.5 family never get the newer `_20260209` / `_20260120` variants. Those newer variants add dynamic filtering (Claude writes code to rank/filter web results before they enter context) and REPL state persistence + programmatic tool calling inside the sandbox, which is what the user-facing pills are supposed to expose. Hardcoding the legacy versions also breaks if a future model family drops the legacy types: the request 400s instead of falling back. Changes: - Add `_anthropic_web_search_version`, `_anthropic_web_fetch_version`, `_anthropic_code_execution_version` helpers that pick the newest variant the model accepts and fall back to the GA versions for everything else. - Add `_ANTHROPIC_CODE_EXECUTION_BETA` constant since the beta header (`code-execution-2025-08-25`) is shared across both code-execution date variants per the upstream docs. - Wire the helpers into `_stream_anthropic` so the outbound body carries the right pinned version per request. - Add parametrized dispatch tests in `test_anthropic_tool_versions.py` covering Opus 4.7/4.6/4.5, Sonnet 4.6/4.5, Haiku 4.5, Opus 4.1/4.0, Sonnet 4.0, 3.5 Sonnet, plus streaming integration tests that verify the outbound body uses the right versions on Opus 4.7 (new web_search + new code_execution), Haiku 4.5 (legacy both), and Sonnet 4.5 (legacy web_search + new code_execution). - Update existing `test_anthropic_code_execution.py` cases that pinned the old version on Opus 4.7 to expect the new ones. Verified end-to-end against the live Anthropic API: Opus 4.7 with both pills enabled accepts the newer-pinned tools without a 400, and Haiku 4.5 still works on the legacy fallback path. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../core/inference/external_provider.py | 100 +++++++-- .../tests/test_anthropic_code_execution.py | 20 +- .../tests/test_anthropic_tool_versions.py | 194 ++++++++++++++++++ 3 files changed, 287 insertions(+), 27 deletions(-) create mode 100644 studio/backend/tests/test_anthropic_tool_versions.py diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 3431c560ba..638312b9db 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -70,6 +70,59 @@ def _anthropic_thinking_spec(model: str) -> Optional[_AnthropicThinkingSpec]: return None +# Anthropic ships date-pinned tool versions per model family. Per the +# tool-reference docs (https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-reference) +# the newer `_20260209` / `_20260120` variants only run on Opus 4.6/4.7 +# and Sonnet 4.6 (web_search / web_fetch) or Opus 4.5+ and Sonnet 4.5+ +# (code_execution). Sending the new versions to an older model returns +# 400 "tool not supported", and sending the old versions on a new model +# misses the dynamic-filtering and free-with-search pricing path. Pick +# the newest combination the model accepts, falling back to the GA +# (`_20250305` / `_20250910` / `_20250825`) defaults for everything else. +_ANTHROPIC_NEW_WEB_PREFIXES = ( + "claude-opus-4-7", + "claude-opus-4-6", + "claude-sonnet-4-6", +) +_ANTHROPIC_NEW_CODE_EXEC_PREFIXES = ( + "claude-opus-4-7", + "claude-opus-4-6", + "claude-sonnet-4-6", + "claude-opus-4-5", + "claude-sonnet-4-5", +) + + +def _anthropic_web_search_version(model: str) -> str: + return ( + "web_search_20260209" + if model.startswith(_ANTHROPIC_NEW_WEB_PREFIXES) + else "web_search_20250305" + ) + + +def _anthropic_web_fetch_version(model: str) -> str: + return ( + "web_fetch_20260209" + if model.startswith(_ANTHROPIC_NEW_WEB_PREFIXES) + else "web_fetch_20250910" + ) + + +def _anthropic_code_execution_version(model: str) -> str: + return ( + "code_execution_20260120" + if model.startswith(_ANTHROPIC_NEW_CODE_EXEC_PREFIXES) + else "code_execution_20250825" + ) + + +# Anthropic's beta-header flag for code execution does NOT change with +# the tool version -- both `_20250825` and `_20260120` are unlocked by +# the same `code-execution-2025-08-25` header per the upstream docs. +_ANTHROPIC_CODE_EXECUTION_BETA = "code-execution-2025-08-25" + + class _MistralThinkingSpec(NamedTuple): models: tuple[str, ...] style: Literal["prompt_mode", "reasoning_effort", "disabled"] @@ -1278,18 +1331,22 @@ class ExternalProviderClient: body["max_tokens"] = budget_tokens + 1024 # Anthropic server-side web_search — see - # https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/web-search-tool - # The tool type is date-pinned (web_search_20250305 today) and - # Anthropic dispatches search calls server-side, returning - # server_tool_use + web_search_tool_result blocks in the SSE - # stream, plus url-citation annotations on text deltas. We - # translate all of that into our local _toolEvent shape so the - # chat UI renders web_search exactly like OpenAI's path. + # https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool + # The tool type is date-pinned per model family. Newer Opus / + # Sonnet 4.6 + 4.7 accept `web_search_20260209` with dynamic + # filtering (Claude writes code to filter results before they + # reach context); everything else uses `web_search_20250305`. + # `_anthropic_web_search_version` picks the right one. Anthropic + # dispatches search calls server-side, returning server_tool_use + # + web_search_tool_result blocks in the SSE stream, plus + # url-citation annotations on text deltas. We translate all of + # that into our local _toolEvent shape so the chat UI renders + # web_search exactly like OpenAI's path. if enabled_tools and "web_search" in enabled_tools: anthropic_tools = list(body.get("tools") or []) anthropic_tools.append( { - "type": "web_search_20250305", + "type": _anthropic_web_search_version(model), "name": "web_search", "max_uses": 5, } @@ -1298,15 +1355,18 @@ class ExternalProviderClient: # Anthropic server-side code execution — see # https://platform.claude.com/docs/en/agents-and-tools/tool-use/code-execution-tool - # `code_execution_20250825` runs Python + bash + str_replace - # file edits inside a 5 GB sandboxed container per request, with - # no internet access. The tool entry itself takes no extra - # parameters; on the SSE stream Anthropic emits two sub-tool - # names — `bash_code_execution` and - # `text_editor_code_execution` — wrapped in the standard - # server_tool_use / *_tool_result block shape. The matching - # beta header (`code-execution-2025-08-25`) is set further down - # in this function alongside the request headers. + # The tool type is date-pinned per model family. + # `_anthropic_code_execution_version` picks `code_execution_20260120` + # for Opus 4.5+ / Sonnet 4.5+ / Opus 4.7 / Sonnet 4.6 (adds REPL + # state persistence + programmatic tool calling) and falls back + # to `code_execution_20250825` everywhere else. Both versions + # run Python + bash + str_replace file edits inside a 5 GB + # sandboxed container per request, with no internet access, and + # both are unlocked by the same `code-execution-2025-08-25` + # `anthropic-beta` header set further down. On the SSE stream + # Anthropic emits two sub-tool names -- `bash_code_execution` + # and `text_editor_code_execution` -- wrapped in the standard + # server_tool_use / *_tool_result block shape. # v1 wires the tool only; file uploads (container_upload # content blocks and generated-file retrieval via the Files # API) are a deliberate follow-up. @@ -1317,7 +1377,7 @@ class ExternalProviderClient: anthropic_tools = list(body.get("tools") or []) anthropic_tools.append( { - "type": "code_execution_20250825", + "type": _anthropic_code_execution_version(model), "name": "code_execution", } ) @@ -1378,8 +1438,8 @@ class ExternalProviderClient: if existing_beta else [] ) - if "code-execution-2025-08-25" not in beta_parts: - beta_parts.append("code-execution-2025-08-25") + if _ANTHROPIC_CODE_EXECUTION_BETA not in beta_parts: + beta_parts.append(_ANTHROPIC_CODE_EXECUTION_BETA) request_headers["anthropic-beta"] = ",".join(beta_parts) try: diff --git a/studio/backend/tests/test_anthropic_code_execution.py b/studio/backend/tests/test_anthropic_code_execution.py index b427ad2c0b..7f6fe58329 100644 --- a/studio/backend/tests/test_anthropic_code_execution.py +++ b/studio/backend/tests/test_anthropic_code_execution.py @@ -116,13 +116,16 @@ def test_code_execution_tool_appended_to_request_body(monkeypatch): body = captured["body"] tools = body.get("tools") or [] + # Opus 4.7 gets the newer date-pinned variant (`_20260120`) that + # supports REPL state persistence + programmatic tool calling. assert { - "type": "code_execution_20250825", + "type": "code_execution_20260120", "name": "code_execution", } in tools # No web_search entry when only code_execution is enabled. - assert all(t.get("type") != "web_search_20250305" for t in tools) - # Beta header carries the documented flag. + assert all("web_search" not in (t.get("type") or "") for t in tools) + # Beta header still carries the documented flag; both `_20250825` + # and `_20260120` are unlocked by the same header per upstream docs. beta_header = captured["headers"].get("anthropic-beta", "") assert "code-execution-2025-08-25" in beta_header @@ -158,8 +161,9 @@ def test_code_execution_with_web_search_sends_both_tools(monkeypatch): tools = captured["body"].get("tools") or [] tool_types = {t.get("type") for t in tools if isinstance(t, dict)} - assert "web_search_20250305" in tool_types - assert "code_execution_20250825" in tool_types + # Opus 4.7 picks the newer pinned versions for both tools. + assert "web_search_20260209" in tool_types + assert "code_execution_20260120" in tool_types assert "code-execution-2025-08-25" in captured["headers"].get("anthropic-beta", "") @@ -192,9 +196,11 @@ def test_no_code_execution_tool_when_pill_off(monkeypatch): _drive(run()) tools = captured["body"].get("tools") or [] - assert all(t.get("type") != "code_execution_20250825" for t in tools) + # Pill off -- neither the legacy nor the new code_execution variant + # may appear on the wire. + assert all("code_execution" not in (t.get("type") or "") for t in tools) # Beta header must NOT mention code-execution when the tool isn't on - # — that flag is opt-in only. + # -- that flag is opt-in only. assert "code-execution-2025-08-25" not in captured["headers"].get( "anthropic-beta", "" ) diff --git a/studio/backend/tests/test_anthropic_tool_versions.py b/studio/backend/tests/test_anthropic_tool_versions.py new file mode 100644 index 0000000000..608977ab07 --- /dev/null +++ b/studio/backend/tests/test_anthropic_tool_versions.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +""" +Unit tests for the per-model Anthropic server-side tool-version dispatch +helpers in ``core.inference.external_provider``. + +Anthropic ships date-pinned tool versions per model family. The newer +``_20260209`` web_search / web_fetch and ``_20260120`` code_execution +variants only run on a subset of models; sending them to an older +model returns a 400 from upstream, and sending the older +``_20250305`` / ``_20250910`` / ``_20250825`` variants to a newer +model misses dynamic filtering and the free-when-paired pricing. The +helpers below decide which version goes out per model; this test pins +the dispatch matrix so future model launches keep working without +silently regressing the newer-version path. + +Covers: +- ``_anthropic_web_search_version`` / ``_anthropic_web_fetch_version`` + pick ``_20260209`` for Opus 4.6+, Opus 4.7, Sonnet 4.6 and fall back + to ``_20250305`` / ``_20250910`` for everything else (4.5 family, + Haiku 4.5, 4.1, 4.0). +- ``_anthropic_code_execution_version`` picks ``_20260120`` for the + Opus 4.5+ / Sonnet 4.5+ / Opus 4.7 / Sonnet 4.6 family and falls back + to ``_20250825`` everywhere else (Haiku 4.5, 4.1, 4.0). +- ``_stream_anthropic`` body integration: when ``enabled_tools= + ["web_search", "code_execution"]`` is set on Opus 4.7, the outbound + body carries the newer pinned versions; the same payload on Haiku + 4.5 falls back to the legacy versions. +- The ``anthropic-beta: code-execution-2025-08-25`` header is sent + unchanged for both code-execution variants (no header rev needed). +""" + +import asyncio +import json + +import httpx +import pytest + +from core.inference import external_provider as ep_mod +from core.inference.external_provider import ( + ExternalProviderClient, + _anthropic_code_execution_version, + _anthropic_web_fetch_version, + _anthropic_web_search_version, +) + + +# ── helper-level dispatch matrix ──────────────────────────────────── + + +@pytest.mark.parametrize( + "model,expected", + [ + ("claude-opus-4-7", "web_search_20260209"), + ("claude-opus-4-6", "web_search_20260209"), + ("claude-sonnet-4-6", "web_search_20260209"), + ("claude-opus-4-5-20251101", "web_search_20250305"), + ("claude-sonnet-4-5-20250929", "web_search_20250305"), + ("claude-haiku-4-5-20251001", "web_search_20250305"), + ("claude-opus-4-1-20250805", "web_search_20250305"), + ("claude-opus-4-20250514", "web_search_20250305"), + ("claude-sonnet-4-20250514", "web_search_20250305"), + ("claude-3-5-sonnet-20241022", "web_search_20250305"), + ], +) +def test_web_search_version_dispatch(model, expected): + assert _anthropic_web_search_version(model) == expected + + +@pytest.mark.parametrize( + "model,expected", + [ + ("claude-opus-4-7", "web_fetch_20260209"), + ("claude-opus-4-6", "web_fetch_20260209"), + ("claude-sonnet-4-6", "web_fetch_20260209"), + ("claude-opus-4-5-20251101", "web_fetch_20250910"), + ("claude-sonnet-4-5-20250929", "web_fetch_20250910"), + ("claude-haiku-4-5-20251001", "web_fetch_20250910"), + ("claude-opus-4-1-20250805", "web_fetch_20250910"), + ], +) +def test_web_fetch_version_dispatch(model, expected): + assert _anthropic_web_fetch_version(model) == expected + + +@pytest.mark.parametrize( + "model,expected", + [ + ("claude-opus-4-7", "code_execution_20260120"), + ("claude-opus-4-6", "code_execution_20260120"), + ("claude-sonnet-4-6", "code_execution_20260120"), + ("claude-opus-4-5-20251101", "code_execution_20260120"), + ("claude-sonnet-4-5-20250929", "code_execution_20260120"), + # Haiku 4.5 only lists the legacy version in the model table. + ("claude-haiku-4-5-20251001", "code_execution_20250825"), + ("claude-opus-4-1-20250805", "code_execution_20250825"), + # Deprecated 4.0 lineage still works on the legacy version. + ("claude-opus-4-20250514", "code_execution_20250825"), + ("claude-sonnet-4-20250514", "code_execution_20250825"), + ], +) +def test_code_execution_version_dispatch(model, expected): + assert _anthropic_code_execution_version(model) == expected + + +# ── streaming integration: outbound body carries the right versions ── + + +def _drive(coro): + return asyncio.new_event_loop().run_until_complete(coro) + + +def _mock_http_client(monkeypatch, handler): + monkeypatch.setattr( + ep_mod, + "_http_client", + httpx.AsyncClient(transport = httpx.MockTransport(handler)), + ) + + +def _make_client() -> ExternalProviderClient: + return ExternalProviderClient( + provider_type = "anthropic", + base_url = "https://api.anthropic.com/v1", + api_key = "sk-ant-test", + ) + + +def _capture_outbound(monkeypatch, model: str) -> dict: + captured: dict = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode("utf-8")) + captured["headers"] = dict(request.headers) + return httpx.Response( + 200, + content = b'event: message_stop\ndata: {"type": "message_stop"}\n\n', + headers = {"content-type": "text/event-stream"}, + ) + + _mock_http_client(monkeypatch, handler) + + async def run(): + client = _make_client() + async for _ in client._stream_anthropic( + messages = [{"role": "user", "content": "hi"}], + model = model, + temperature = 0.7, + top_p = 0.95, + max_tokens = 64, + enabled_tools = ["web_search", "code_execution"], + ): + pass + await client.close() + + _drive(run()) + return captured + + +def test_outbound_body_uses_new_versions_on_opus_4_7(monkeypatch): + captured = _capture_outbound(monkeypatch, "claude-opus-4-7") + tool_types = {t.get("type") for t in (captured["body"].get("tools") or [])} + assert "web_search_20260209" in tool_types + assert "code_execution_20260120" in tool_types + assert "web_search_20250305" not in tool_types + assert "code_execution_20250825" not in tool_types + # Beta header for code execution stays on the existing flag for + # both _20250825 and _20260120; the API uses one header to gate + # the feature, not the date. + assert "code-execution-2025-08-25" in captured["headers"].get( + "anthropic-beta", + "", + ) + + +def test_outbound_body_falls_back_on_haiku_4_5(monkeypatch): + captured = _capture_outbound(monkeypatch, "claude-haiku-4-5-20251001") + tool_types = {t.get("type") for t in (captured["body"].get("tools") or [])} + # Haiku 4.5 only accepts the legacy versions. + assert "web_search_20250305" in tool_types + assert "code_execution_20250825" in tool_types + assert "web_search_20260209" not in tool_types + assert "code_execution_20260120" not in tool_types + + +def test_outbound_body_mixes_versions_on_sonnet_4_5(monkeypatch): + # Sonnet 4.5 gets the new code_execution but the old web_search. + captured = _capture_outbound(monkeypatch, "claude-sonnet-4-5-20250929") + tool_types = {t.get("type") for t in (captured["body"].get("tools") or [])} + assert "web_search_20250305" in tool_types + assert "code_execution_20260120" in tool_types + assert "web_search_20260209" not in tool_types + assert "code_execution_20250825" not in tool_types