From 1c21d1d3f4ea98d7a30f5ce16c2337d7d169da12 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 09:37:53 +0000 Subject: [PATCH] Studio: surface every live provider model in the picker Two regressions in the provider registry hide live models from the chat picker: 1. The Anthropic denylist `-\d{8}$` strips every dated id. Per the models overview, the pre-4.6 generation ships only as dated ids (Opus 4.5, Sonnet 4.5, Haiku 4.5, Opus 4.1, the deprecated 4.0 family). The denylist hides 6 of the 9 live models -- a user wanting Haiku 4.5 or Sonnet 4.5 can't pick them from the dropdown. 2. The OpenAI allowlist `^(gpt-5\.[345]|gpt-4\.5|o3)(?:[-.]|$)` silently drops every family OpenAI ships outside the hardcoded set. Existing chat models like gpt-3.5-turbo, gpt-4, gpt-4o, gpt-4.1, gpt-5, gpt-5.1, gpt-5.2, gpt-5.3, o1, o4-mini and anything OpenAI launches next get dropped on the floor. Changes: - Drop the Anthropic denylist entirely; show every live id. - Update Anthropic default_models seed to use the canonical dated ids so the pre-load list matches what /v1/models returns. - Replace the OpenAI allowlist with a non-chat denylist that drops only known non-chat model families: embeddings, TTS, image, moderation, whisper, audio, realtime, transcribe, search-preview, sora video, computer-use harness, legacy bases (babbage, davinci, ada, curie), fine-tunes (ft:*), and dated snapshots. Every chat family auto-surfaces the moment OpenAI lists it. - Add `test_provider_registry_filters.py` with parametrized cases covering current + hypothetical-future chat ids that must survive, plus the full non-chat surface area that must be dropped. Live verified against the real /v1/models on both providers: - Anthropic: 9 -> 9 (was 9 -> 3). - OpenAI: 129 -> 46 chat models (was 129 -> 12). --- studio/backend/core/inference/providers.py | 45 +++++--- .../tests/test_provider_registry_filters.py | 108 ++++++++++++++++++ 2 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 studio/backend/tests/test_provider_registry_filters.py diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index fef9ba3e12..31112f2061 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -26,14 +26,24 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "supports_tool_calling": True, "auth_header": "Authorization", "auth_prefix": "Bearer ", - # Keep the model picker scoped to the current generation. The remote - # /v1/models listing returns dozens of historical snapshots, fine-tunes - # and non-chat models (embeddings, TTS, image, moderation) that we - # never want to surface in the chat UI. Filtering here so backend - # is the single source of truth. - "model_id_allowlist": re.compile(r"^(gpt-5\.[345]|gpt-4\.5|o3)(?:[-.]|$)"), - # Hide dated snapshots and the retired plain gpt-5.3 id. - "model_id_denylist": re.compile(r"^(gpt-5\.3)$|-\d{4}-\d{2}-\d{2}$"), + # The remote /v1/models listing returns the full account catalog, + # including non-chat models (embeddings, TTS, image, moderation, + # whisper, dall-e) and fine-tunes. Previously we used a hardcoded + # family allowlist (`gpt-5\.[345]|gpt-4\.5|o3`) which silently + # dropped every new family OpenAI shipped, including the gpt-5.5 + # generation today. Switch to a non-chat denylist instead so any + # new chat family auto-appears the moment OpenAI lists it. The + # patterns below cover every non-chat model id OpenAI has ever + # published; chat ids never start with these prefixes. + "model_id_denylist": re.compile( + r"(?:^|-)(?:embedding|tts|whisper|moderation|image|search|audio|" + r"realtime|transcribe|babbage|davinci|ada|curie|sora)\b" + r"|^dall-e\b" + r"|^computer-use\b" + r"|^text-(?:embedding|moderation|davinci|curie|babbage|ada)\b" + r"|^ft:" + r"|-\d{4}-\d{2}-\d{2}$" + ), }, "anthropic": { "display_name": "Anthropic", @@ -42,16 +52,17 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "claude-opus-4-7", "claude-opus-4-6", "claude-sonnet-4-6", - "claude-opus-4-5", - "claude-sonnet-4-5", - "claude-haiku-4-5", + "claude-opus-4-5-20251101", + "claude-sonnet-4-5-20250929", + "claude-haiku-4-5-20251001", ], - # Anthropic /v1/models returns dated snapshot ids alongside the - # canonical names (e.g. claude-3-5-sonnet-20241022). Hide the - # YYYYMMDD-suffixed variants from the picker — same intent as the - # OpenAI denylist, just a different date format (no dashes between - # year/month/day). - "model_id_denylist": re.compile(r"-\d{8}$"), + # Anthropic's /v1/models returns dated ids for every model in the + # pre-4.6 generation -- `claude-opus-4-5-20251101`, + # `claude-haiku-4-5-20251001`, `claude-sonnet-4-5-20250929`, + # `claude-opus-4-1-20250805`. Per the models overview, those + # dated ids ARE the canonical names for that generation, not + # snapshots to hide. Dropping the previous `-\d{8}$` denylist + # so every live model the API returns reaches the picker. "supports_streaming": True, "supports_vision": True, "supports_tool_calling": False, diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py new file mode 100644 index 0000000000..d709c4ebba --- /dev/null +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Provider registry model-id filter regression tests. + +The OpenAI ``model_id_allowlist`` previously hardcoded the gpt-5.3/4/5 +families plus gpt-4.5 / o3 -- silently dropping every future family +OpenAI shipped. Anthropic's ``model_id_denylist`` previously stripped +every dated id, hiding the canonical names of every pre-4.6 model +(Opus 4.5, Sonnet 4.5, Haiku 4.5, Opus 4.1, the 4.0 family). + +These tests pin the new non-chat denylist (OpenAI) and the empty +denylist (Anthropic) by walking realistic ``/v1/models`` listings +through ``PROVIDER_REGISTRY`` and asserting the surviving set. +""" + +from core.inference.providers import PROVIDER_REGISTRY + + +def _apply(provider_type: str, candidate_ids: list[str]) -> list[str]: + """Mirror the filter logic in ``routes/providers.list_models``.""" + info = PROVIDER_REGISTRY[provider_type] + out = list(candidate_ids) + allow = info.get("model_id_allowlist") + if allow is not None: + out = [m for m in out if allow.match(m)] + deny = info.get("model_id_denylist") + if deny is not None: + out = [m for m in out if not deny.search(m)] + return out + + +# ── OpenAI: non-chat denylist drops only non-chat ids ────────────── + + +def test_openai_keeps_every_known_chat_family(): + live = [ + # Current generation (must survive). + "gpt-5.5", "gpt-5.5-pro", + "gpt-5.4", "gpt-5.4-pro", "gpt-5.4-mini", "gpt-5.4-nano", + "gpt-5.3-codex", "gpt-5.3-chat-latest", + "o3", "o3-pro", "o3-mini", "o3-deep-research", + # Hypothetical future families that the old allowlist would have + # silently dropped -- they MUST surface under the new denylist. + "gpt-5.6", "gpt-5.6-mini", "gpt-6", "gpt-6-pro", + "o4", "o4-pro", "o5", + ] + surviving = _apply("openai", live) + assert surviving == live, surviving + + +def test_openai_drops_non_chat_ids(): + noise = [ + # Embeddings / TTS / image / moderation / whisper / audio etc. + "text-embedding-3-small", "text-embedding-3-large", + "text-embedding-ada-002", + "text-moderation-latest", "text-moderation-stable", + "tts-1", "tts-1-hd", "gpt-4o-tts", + "whisper-1", + "dall-e-2", "dall-e-3", + "gpt-image-1", "gpt-image-2", "gpt-image-1-mini", + "chatgpt-image-latest", + "gpt-audio-1.5", "gpt-realtime-2", "gpt-4o-realtime-preview", + "gpt-4o-transcribe", "gpt-4o-search-preview", + "gpt-4o-mini-search-preview", "gpt-4o-mini-transcribe", + "gpt-4o-mini-tts", + "omni-moderation-latest", + # Video generation. + "sora-2", "sora-2-pro", + # Computer-use is an agentic harness, not a chat id. + "computer-use-preview", + # Legacy bases. + "babbage-002", "davinci-002", "text-davinci-003", + "text-curie-001", "text-ada-001", + # Fine-tunes. + "ft:gpt-4o-mini:acme:abc:xyz", + # Dated snapshots are still hidden. + "gpt-4o-2024-08-06", "gpt-4o-mini-2024-07-18", + "gpt-5.5-2026-04-23", + ] + surviving = _apply("openai", noise) + assert surviving == [], surviving + + +# ── Anthropic: empty denylist; dated ids ARE canonical ─────────────── + + +def test_anthropic_surfaces_every_live_model_including_dated_ids(): + # The full set of ids /v1/models returns today. + live = [ + "claude-opus-4-7", + "claude-sonnet-4-6", + "claude-opus-4-6", + "claude-opus-4-5-20251101", + "claude-sonnet-4-5-20250929", + "claude-haiku-4-5-20251001", + "claude-opus-4-1-20250805", + "claude-opus-4-20250514", + "claude-sonnet-4-20250514", + ] + surviving = _apply("anthropic", live) + assert surviving == live, surviving + + +def test_anthropic_default_models_match_filter(): + info = PROVIDER_REGISTRY["anthropic"] + surviving = _apply("anthropic", list(info["default_models"])) + assert surviving == list(info["default_models"]), surviving