From 1c21d1d3f4ea98d7a30f5ce16c2337d7d169da12 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 09:37:53 +0000 Subject: [PATCH 01/13] Studio: surface every live provider model in the picker Two regressions in the provider registry hide live models from the chat picker: 1. The Anthropic denylist `-\d{8}$` strips every dated id. Per the models overview, the pre-4.6 generation ships only as dated ids (Opus 4.5, Sonnet 4.5, Haiku 4.5, Opus 4.1, the deprecated 4.0 family). The denylist hides 6 of the 9 live models -- a user wanting Haiku 4.5 or Sonnet 4.5 can't pick them from the dropdown. 2. The OpenAI allowlist `^(gpt-5\.[345]|gpt-4\.5|o3)(?:[-.]|$)` silently drops every family OpenAI ships outside the hardcoded set. Existing chat models like gpt-3.5-turbo, gpt-4, gpt-4o, gpt-4.1, gpt-5, gpt-5.1, gpt-5.2, gpt-5.3, o1, o4-mini and anything OpenAI launches next get dropped on the floor. Changes: - Drop the Anthropic denylist entirely; show every live id. - Update Anthropic default_models seed to use the canonical dated ids so the pre-load list matches what /v1/models returns. - Replace the OpenAI allowlist with a non-chat denylist that drops only known non-chat model families: embeddings, TTS, image, moderation, whisper, audio, realtime, transcribe, search-preview, sora video, computer-use harness, legacy bases (babbage, davinci, ada, curie), fine-tunes (ft:*), and dated snapshots. Every chat family auto-surfaces the moment OpenAI lists it. - Add `test_provider_registry_filters.py` with parametrized cases covering current + hypothetical-future chat ids that must survive, plus the full non-chat surface area that must be dropped. Live verified against the real /v1/models on both providers: - Anthropic: 9 -> 9 (was 9 -> 3). - OpenAI: 129 -> 46 chat models (was 129 -> 12). --- studio/backend/core/inference/providers.py | 45 +++++--- .../tests/test_provider_registry_filters.py | 108 ++++++++++++++++++ 2 files changed, 136 insertions(+), 17 deletions(-) create mode 100644 studio/backend/tests/test_provider_registry_filters.py diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index fef9ba3e12..31112f2061 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -26,14 +26,24 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "supports_tool_calling": True, "auth_header": "Authorization", "auth_prefix": "Bearer ", - # Keep the model picker scoped to the current generation. The remote - # /v1/models listing returns dozens of historical snapshots, fine-tunes - # and non-chat models (embeddings, TTS, image, moderation) that we - # never want to surface in the chat UI. Filtering here so backend - # is the single source of truth. - "model_id_allowlist": re.compile(r"^(gpt-5\.[345]|gpt-4\.5|o3)(?:[-.]|$)"), - # Hide dated snapshots and the retired plain gpt-5.3 id. - "model_id_denylist": re.compile(r"^(gpt-5\.3)$|-\d{4}-\d{2}-\d{2}$"), + # The remote /v1/models listing returns the full account catalog, + # including non-chat models (embeddings, TTS, image, moderation, + # whisper, dall-e) and fine-tunes. Previously we used a hardcoded + # family allowlist (`gpt-5\.[345]|gpt-4\.5|o3`) which silently + # dropped every new family OpenAI shipped, including the gpt-5.5 + # generation today. Switch to a non-chat denylist instead so any + # new chat family auto-appears the moment OpenAI lists it. The + # patterns below cover every non-chat model id OpenAI has ever + # published; chat ids never start with these prefixes. + "model_id_denylist": re.compile( + r"(?:^|-)(?:embedding|tts|whisper|moderation|image|search|audio|" + r"realtime|transcribe|babbage|davinci|ada|curie|sora)\b" + r"|^dall-e\b" + r"|^computer-use\b" + r"|^text-(?:embedding|moderation|davinci|curie|babbage|ada)\b" + r"|^ft:" + r"|-\d{4}-\d{2}-\d{2}$" + ), }, "anthropic": { "display_name": "Anthropic", @@ -42,16 +52,17 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "claude-opus-4-7", "claude-opus-4-6", "claude-sonnet-4-6", - "claude-opus-4-5", - "claude-sonnet-4-5", - "claude-haiku-4-5", + "claude-opus-4-5-20251101", + "claude-sonnet-4-5-20250929", + "claude-haiku-4-5-20251001", ], - # Anthropic /v1/models returns dated snapshot ids alongside the - # canonical names (e.g. claude-3-5-sonnet-20241022). Hide the - # YYYYMMDD-suffixed variants from the picker — same intent as the - # OpenAI denylist, just a different date format (no dashes between - # year/month/day). - "model_id_denylist": re.compile(r"-\d{8}$"), + # Anthropic's /v1/models returns dated ids for every model in the + # pre-4.6 generation -- `claude-opus-4-5-20251101`, + # `claude-haiku-4-5-20251001`, `claude-sonnet-4-5-20250929`, + # `claude-opus-4-1-20250805`. Per the models overview, those + # dated ids ARE the canonical names for that generation, not + # snapshots to hide. Dropping the previous `-\d{8}$` denylist + # so every live model the API returns reaches the picker. "supports_streaming": True, "supports_vision": True, "supports_tool_calling": False, diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py new file mode 100644 index 0000000000..d709c4ebba --- /dev/null +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 + +"""Provider registry model-id filter regression tests. + +The OpenAI ``model_id_allowlist`` previously hardcoded the gpt-5.3/4/5 +families plus gpt-4.5 / o3 -- silently dropping every future family +OpenAI shipped. Anthropic's ``model_id_denylist`` previously stripped +every dated id, hiding the canonical names of every pre-4.6 model +(Opus 4.5, Sonnet 4.5, Haiku 4.5, Opus 4.1, the 4.0 family). + +These tests pin the new non-chat denylist (OpenAI) and the empty +denylist (Anthropic) by walking realistic ``/v1/models`` listings +through ``PROVIDER_REGISTRY`` and asserting the surviving set. +""" + +from core.inference.providers import PROVIDER_REGISTRY + + +def _apply(provider_type: str, candidate_ids: list[str]) -> list[str]: + """Mirror the filter logic in ``routes/providers.list_models``.""" + info = PROVIDER_REGISTRY[provider_type] + out = list(candidate_ids) + allow = info.get("model_id_allowlist") + if allow is not None: + out = [m for m in out if allow.match(m)] + deny = info.get("model_id_denylist") + if deny is not None: + out = [m for m in out if not deny.search(m)] + return out + + +# ── OpenAI: non-chat denylist drops only non-chat ids ────────────── + + +def test_openai_keeps_every_known_chat_family(): + live = [ + # Current generation (must survive). + "gpt-5.5", "gpt-5.5-pro", + "gpt-5.4", "gpt-5.4-pro", "gpt-5.4-mini", "gpt-5.4-nano", + "gpt-5.3-codex", "gpt-5.3-chat-latest", + "o3", "o3-pro", "o3-mini", "o3-deep-research", + # Hypothetical future families that the old allowlist would have + # silently dropped -- they MUST surface under the new denylist. + "gpt-5.6", "gpt-5.6-mini", "gpt-6", "gpt-6-pro", + "o4", "o4-pro", "o5", + ] + surviving = _apply("openai", live) + assert surviving == live, surviving + + +def test_openai_drops_non_chat_ids(): + noise = [ + # Embeddings / TTS / image / moderation / whisper / audio etc. + "text-embedding-3-small", "text-embedding-3-large", + "text-embedding-ada-002", + "text-moderation-latest", "text-moderation-stable", + "tts-1", "tts-1-hd", "gpt-4o-tts", + "whisper-1", + "dall-e-2", "dall-e-3", + "gpt-image-1", "gpt-image-2", "gpt-image-1-mini", + "chatgpt-image-latest", + "gpt-audio-1.5", "gpt-realtime-2", "gpt-4o-realtime-preview", + "gpt-4o-transcribe", "gpt-4o-search-preview", + "gpt-4o-mini-search-preview", "gpt-4o-mini-transcribe", + "gpt-4o-mini-tts", + "omni-moderation-latest", + # Video generation. + "sora-2", "sora-2-pro", + # Computer-use is an agentic harness, not a chat id. + "computer-use-preview", + # Legacy bases. + "babbage-002", "davinci-002", "text-davinci-003", + "text-curie-001", "text-ada-001", + # Fine-tunes. + "ft:gpt-4o-mini:acme:abc:xyz", + # Dated snapshots are still hidden. + "gpt-4o-2024-08-06", "gpt-4o-mini-2024-07-18", + "gpt-5.5-2026-04-23", + ] + surviving = _apply("openai", noise) + assert surviving == [], surviving + + +# ── Anthropic: empty denylist; dated ids ARE canonical ─────────────── + + +def test_anthropic_surfaces_every_live_model_including_dated_ids(): + # The full set of ids /v1/models returns today. + live = [ + "claude-opus-4-7", + "claude-sonnet-4-6", + "claude-opus-4-6", + "claude-opus-4-5-20251101", + "claude-sonnet-4-5-20250929", + "claude-haiku-4-5-20251001", + "claude-opus-4-1-20250805", + "claude-opus-4-20250514", + "claude-sonnet-4-20250514", + ] + surviving = _apply("anthropic", live) + assert surviving == live, surviving + + +def test_anthropic_default_models_match_filter(): + info = PROVIDER_REGISTRY["anthropic"] + surviving = _apply("anthropic", list(info["default_models"])) + assert surviving == list(info["default_models"]), surviving From 4434e1eeb7026e1ac130576ae961cf703763b3d2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 09:38:31 +0000 Subject: [PATCH 02/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../tests/test_provider_registry_filters.py | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index d709c4ebba..66c90027e2 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -36,14 +36,27 @@ def _apply(provider_type: str, candidate_ids: list[str]) -> list[str]: def test_openai_keeps_every_known_chat_family(): live = [ # Current generation (must survive). - "gpt-5.5", "gpt-5.5-pro", - "gpt-5.4", "gpt-5.4-pro", "gpt-5.4-mini", "gpt-5.4-nano", - "gpt-5.3-codex", "gpt-5.3-chat-latest", - "o3", "o3-pro", "o3-mini", "o3-deep-research", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.4", + "gpt-5.4-pro", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.3-codex", + "gpt-5.3-chat-latest", + "o3", + "o3-pro", + "o3-mini", + "o3-deep-research", # Hypothetical future families that the old allowlist would have # silently dropped -- they MUST surface under the new denylist. - "gpt-5.6", "gpt-5.6-mini", "gpt-6", "gpt-6-pro", - "o4", "o4-pro", "o5", + "gpt-5.6", + "gpt-5.6-mini", + "gpt-6", + "gpt-6-pro", + "o4", + "o4-pro", + "o5", ] surviving = _apply("openai", live) assert surviving == live, surviving @@ -52,30 +65,46 @@ def test_openai_keeps_every_known_chat_family(): def test_openai_drops_non_chat_ids(): noise = [ # Embeddings / TTS / image / moderation / whisper / audio etc. - "text-embedding-3-small", "text-embedding-3-large", + "text-embedding-3-small", + "text-embedding-3-large", "text-embedding-ada-002", - "text-moderation-latest", "text-moderation-stable", - "tts-1", "tts-1-hd", "gpt-4o-tts", + "text-moderation-latest", + "text-moderation-stable", + "tts-1", + "tts-1-hd", + "gpt-4o-tts", "whisper-1", - "dall-e-2", "dall-e-3", - "gpt-image-1", "gpt-image-2", "gpt-image-1-mini", + "dall-e-2", + "dall-e-3", + "gpt-image-1", + "gpt-image-2", + "gpt-image-1-mini", "chatgpt-image-latest", - "gpt-audio-1.5", "gpt-realtime-2", "gpt-4o-realtime-preview", - "gpt-4o-transcribe", "gpt-4o-search-preview", - "gpt-4o-mini-search-preview", "gpt-4o-mini-transcribe", + "gpt-audio-1.5", + "gpt-realtime-2", + "gpt-4o-realtime-preview", + "gpt-4o-transcribe", + "gpt-4o-search-preview", + "gpt-4o-mini-search-preview", + "gpt-4o-mini-transcribe", "gpt-4o-mini-tts", "omni-moderation-latest", # Video generation. - "sora-2", "sora-2-pro", + "sora-2", + "sora-2-pro", # Computer-use is an agentic harness, not a chat id. "computer-use-preview", # Legacy bases. - "babbage-002", "davinci-002", "text-davinci-003", - "text-curie-001", "text-ada-001", + "babbage-002", + "davinci-002", + "text-davinci-003", + "text-curie-001", + "text-ada-001", # Fine-tunes. "ft:gpt-4o-mini:acme:abc:xyz", # Dated snapshots are still hidden. - "gpt-4o-2024-08-06", "gpt-4o-mini-2024-07-18", + "gpt-4o-2024-08-06", + "gpt-4o-mini-2024-07-18", "gpt-5.5-2026-04-23", ] surviving = _apply("openai", noise) From 9a9c6b09496e854655fb3f386523687a5817d9b6 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 11:06:17 +0000 Subject: [PATCH 03/13] Address review: tighten denylist + document mid-id matching intent Two tweaks to the OpenAI model_id_denylist after review: 1. `(?:^|-)search\b` was a false positive: it dropped `gpt-4o-search-preview` and `gpt-4o-mini-search-preview`, both of which are chat-with-retrieval models that respond to standard /v1/chat/completions and should appear in the picker. Replaced with a `-search-api(?:-\d{4}-\d{2}-\d{2})?$` suffix match that only catches the standalone search-API endpoint (`gpt-5-search-api`, `gpt-5-search-api-2025-10-14`). 2. The legacy completion bases (babbage / davinci / ada / curie) had `(?:^|-)` anchoring, which would mid-id-match a hypothetical future chat id like `gpt-7-davinci-edition`. Tightened to `^(?:babbage|davinci|ada|curie)\b` so only the legacy ids (`babbage-002`, `davinci-002`, `text-davinci-003`, ...) are dropped while leaving any future chat id containing the word alone. Expanded the regex docstring to spell out the intent per pattern group (feature-suffix matches are intentional mid-id; legacy bases are ^-anchored; standalone families are prefix-only) so a future maintainer doesn't undo it. Tests: added test_openai_search_preview_is_kept_search_api_is_dropped to pin the chat-vs-API distinction, and test_openai_legacy_completion_names_only_match_at_id_start to lock in the mid-id safety on the legacy-base set. --- studio/backend/core/inference/providers.py | 47 +++++++++++-- .../tests/test_provider_registry_filters.py | 66 ++++++++++++++++++- 2 files changed, 105 insertions(+), 8 deletions(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 31112f2061..bebebdc396 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -32,16 +32,51 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # family allowlist (`gpt-5\.[345]|gpt-4\.5|o3`) which silently # dropped every new family OpenAI shipped, including the gpt-5.5 # generation today. Switch to a non-chat denylist instead so any - # new chat family auto-appears the moment OpenAI lists it. The - # patterns below cover every non-chat model id OpenAI has ever - # published; chat ids never start with these prefixes. + # new chat family auto-appears the moment OpenAI lists it. + # + # Pattern strategy: + # - Unambiguous *feature* indicators (tts/whisper/transcribe/audio/ + # realtime/image/embedding/moderation/sora) match anywhere in + # the id with `(?:^|-)` because OpenAI uses them as the + # primary qualifier on every variant -- e.g. canonical + # `tts-1` AND family variants `gpt-4o-tts`, `gpt-4o-mini-tts`. + # These words don't appear in legitimate chat model ids + # mid-string, so the mid-id match is intentional and safe. + # - `search` is INTENTIONALLY NOT in the mid-id set because + # `gpt-4o-search-preview` and `gpt-4o-mini-search-preview` are + # chat-with-retrieval models that absolutely belong in the + # picker. The standalone search API is caught separately by + # `-search-api` (suffix) which never appears on a chat id. + # - Legacy completion bases (babbage / davinci / ada / curie) + # are ^-anchored: they ONLY ever begin a legacy id like + # `babbage-002`, `davinci-002`, `text-davinci-003`. A future + # hypothetical `gpt-7-davinci-edition` chat model would NOT + # be dropped, which is the right default for a denylist. + # - Prefix-only patterns (`^dall-e`, `^computer-use`, `^ft:`, + # `^gpt-image`) cover canonical non-chat families without + # risk of mid-id false positives. + # + # Verified against the live /v1/models listing 2026-05-22. "model_id_denylist": re.compile( - r"(?:^|-)(?:embedding|tts|whisper|moderation|image|search|audio|" - r"realtime|transcribe|babbage|davinci|ada|curie|sora)\b" + # Feature suffixes that mark a non-chat variant on any base. + r"(?:^|-)(?:embedding|tts|whisper|moderation|image|audio|" + r"realtime|transcribe|sora)\b" + # Legacy completion bases -- ^-anchored to avoid false + # positives on hypothetical future chat ids containing + # those words mid-string. + r"|^(?:babbage|davinci|ada|curie)\b" + r"|^text-(?:embedding|moderation|davinci|curie|babbage|ada)\b" + # Standalone search API (separate endpoint shape). + # `gpt-4o-search-preview` (chat-with-search) is intentionally + # NOT matched here. + r"|-search-api(?:-\d{4}-\d{2}-\d{2})?$" + # Canonical non-chat prefixes. r"|^dall-e\b" r"|^computer-use\b" - r"|^text-(?:embedding|moderation|davinci|curie|babbage|ada)\b" + # Fine-tunes carry the user's tenant in the id. r"|^ft:" + # Dated snapshot suffixes hide behind the canonical id + # which is also in the listing. r"|-\d{4}-\d{2}-\d{2}$" ), }, diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index 66c90027e2..8c20d12173 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -48,6 +48,14 @@ def test_openai_keeps_every_known_chat_family(): "o3-pro", "o3-mini", "o3-deep-research", + # `gpt-4o-search-preview` and the mini variant are chat-with- + # retrieval models that respond to standard /v1/chat/completions + # and absolutely belong in the picker. The previous regex + # dropped them as a false positive of `(?:^|-)search`; the + # tightened denylist intentionally only catches the standalone + # search-api suffix. + "gpt-4o-search-preview", + "gpt-4o-mini-search-preview", # Hypothetical future families that the old allowlist would have # silently dropped -- they MUST surface under the new denylist. "gpt-5.6", @@ -84,11 +92,13 @@ def test_openai_drops_non_chat_ids(): "gpt-realtime-2", "gpt-4o-realtime-preview", "gpt-4o-transcribe", - "gpt-4o-search-preview", - "gpt-4o-mini-search-preview", "gpt-4o-mini-transcribe", "gpt-4o-mini-tts", "omni-moderation-latest", + # Standalone search API endpoint (distinct from the + # `*-search-preview` chat models above). + "gpt-5-search-api", + "gpt-5-search-api-2025-10-14", # Video generation. "sora-2", "sora-2-pro", @@ -111,6 +121,58 @@ def test_openai_drops_non_chat_ids(): assert surviving == [], surviving +def test_openai_search_preview_is_kept_search_api_is_dropped(): + # Pin the search-vs-search-api distinction so a future regex tweak + # doesn't silently regress to dropping chat-with-search models. + kept = _apply( + "openai", + [ + "gpt-4o-search-preview", + "gpt-4o-mini-search-preview", + ], + ) + assert kept == [ + "gpt-4o-search-preview", + "gpt-4o-mini-search-preview", + ], kept + + dropped = _apply( + "openai", + [ + "gpt-5-search-api", + "gpt-5-search-api-2025-10-14", + ], + ) + assert dropped == [], dropped + + +def test_openai_legacy_completion_names_only_match_at_id_start(): + # Hypothetical future chat ids that happen to contain a legacy + # completion-family name mid-string must NOT be dropped. The + # `^(?:babbage|davinci|ada|curie)\b` anchor is what makes this safe. + kept = _apply( + "openai", + [ + "gpt-7-davinci-edition", + "gpt-7-ada-chat", + "gpt-7-curie-pro", + "gpt-7-babbage-mini", + ], + ) + assert kept == [ + "gpt-7-davinci-edition", + "gpt-7-ada-chat", + "gpt-7-curie-pro", + "gpt-7-babbage-mini", + ], kept + # ...but the actual legacy-base ids stay dropped. + dropped = _apply( + "openai", + ["babbage-002", "davinci-002", "text-davinci-003"], + ) + assert dropped == [], dropped + + # ── Anthropic: empty denylist; dated ids ARE canonical ─────────────── From c51526d244ab352c4ec38bea0abd8e3af92f0ab8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 11:38:30 +0000 Subject: [PATCH 04/13] Address review: keep gpt-audio family in the picker Codex P2 caught that `(?:^|-)audio` mid-id matching dropped `gpt-audio`, `gpt-audio-mini`, `gpt-audio-1.5` -- but those are chat-completion-capable models per OpenAI's audio guide (text in / text or audio out via /v1/chat/completions and /v1/responses). The denylist was silently hiding supported chat options. Removed `audio` from the mid-id feature suffix set. The narrower non-chat audio endpoints stay dropped: - `tts-*` / `gpt-*-tts`: text-to-speech only - `whisper-*`: STT only - `gpt-*-transcribe`: /v1/audio/transcriptions only - `gpt-realtime*`: /v1/realtime (WebSocket) endpoint, not chat Updated the regex comment to spell out the chat/non-chat split per family. Existing test added gpt-audio* to the keep list and moved gpt-audio-1.5 out of the drop list; new test test_openai_audio_family_is_chat_capable_and_kept pins the contract so a future regex tweak can't silently undo it. --- studio/backend/core/inference/providers.py | 19 +++++++-- .../tests/test_provider_registry_filters.py | 40 ++++++++++++++++++- 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index bebebdc396..850566757f 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -35,13 +35,23 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # new chat family auto-appears the moment OpenAI lists it. # # Pattern strategy: - # - Unambiguous *feature* indicators (tts/whisper/transcribe/audio/ - # realtime/image/embedding/moderation/sora) match anywhere in - # the id with `(?:^|-)` because OpenAI uses them as the + # - Unambiguous *feature* indicators (tts/whisper/transcribe/ + # realtime/image/embedding/moderation/sora) match anywhere + # in the id with `(?:^|-)` because OpenAI uses them as the # primary qualifier on every variant -- e.g. canonical # `tts-1` AND family variants `gpt-4o-tts`, `gpt-4o-mini-tts`. # These words don't appear in legitimate chat model ids # mid-string, so the mid-id match is intentional and safe. + # - `audio` is INTENTIONALLY NOT in the mid-id set: the + # `gpt-audio` / `gpt-audio-mini` / `gpt-audio-1.5` family is + # chat-completion-capable (text in / text or audio out via + # /v1/chat/completions and /v1/responses), so dropping them + # would hide supported chat models from the picker. The + # audio-only endpoint families (`tts-*`, `whisper-*`, + # `*-transcribe`) ARE caught above. + # - `realtime` keeps the mid-id match because the + # `gpt-realtime*` family is a separate /v1/realtime + # (WebSocket) endpoint, not chat-completions. # - `search` is INTENTIONALLY NOT in the mid-id set because # `gpt-4o-search-preview` and `gpt-4o-mini-search-preview` are # chat-with-retrieval models that absolutely belong in the @@ -59,7 +69,8 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # Verified against the live /v1/models listing 2026-05-22. "model_id_denylist": re.compile( # Feature suffixes that mark a non-chat variant on any base. - r"(?:^|-)(?:embedding|tts|whisper|moderation|image|audio|" + # Note: `audio` is omitted on purpose -- see docstring above. + r"(?:^|-)(?:embedding|tts|whisper|moderation|image|" r"realtime|transcribe|sora)\b" # Legacy completion bases -- ^-anchored to avoid false # positives on hypothetical future chat ids containing diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index 8c20d12173..c055739c30 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -56,6 +56,13 @@ def test_openai_keeps_every_known_chat_family(): # search-api suffix. "gpt-4o-search-preview", "gpt-4o-mini-search-preview", + # gpt-audio family: chat-completion-capable (text in / text or + # audio out) per OpenAI's audio guide. Earlier revisions of + # this denylist dropped them via `(?:^|-)audio`; the regex + # comment block now keeps them. + "gpt-audio", + "gpt-audio-mini", + "gpt-audio-1.5", # Hypothetical future families that the old allowlist would have # silently dropped -- they MUST surface under the new denylist. "gpt-5.6", @@ -70,9 +77,39 @@ def test_openai_keeps_every_known_chat_family(): assert surviving == live, surviving +def test_openai_audio_family_is_chat_capable_and_kept(): + # Pin the chat/non-chat split that lives in the regex comment: + # gpt-audio is kept (chat completions accept it), gpt-realtime is + # dropped (separate /v1/realtime endpoint), gpt-4o-transcribe is + # dropped (audio-only input via /v1/audio/transcriptions). + kept = _apply( + "openai", + [ + "gpt-audio", + "gpt-audio-1.5", + "gpt-audio-mini", + ], + ) + assert kept == ["gpt-audio", "gpt-audio-1.5", "gpt-audio-mini"], kept + + dropped = _apply( + "openai", + [ + "gpt-realtime", + "gpt-realtime-mini", + "gpt-4o-realtime-preview", + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + ], + ) + assert dropped == [], dropped + + def test_openai_drops_non_chat_ids(): noise = [ - # Embeddings / TTS / image / moderation / whisper / audio etc. + # Embeddings / TTS / image / moderation / whisper / etc. + # gpt-audio* is intentionally OMITTED from this list -- it is + # chat-capable. See test_openai_audio_family_is_chat_capable_and_kept. "text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002", @@ -88,7 +125,6 @@ def test_openai_drops_non_chat_ids(): "gpt-image-2", "gpt-image-1-mini", "chatgpt-image-latest", - "gpt-audio-1.5", "gpt-realtime-2", "gpt-4o-realtime-preview", "gpt-4o-transcribe", From 6ea20c5aa7c4241b1c65651939893a6791c4dfd9 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 12:34:38 +0000 Subject: [PATCH 05/13] Address review: keep gpt-realtime family in the picker Codex P1 follow-up to the audio-family fix: gpt-realtime / gpt-realtime-mini / gpt-realtime-1.5 / gpt-realtime-2 are also chat-completion-capable today per OpenAI's realtime guide. They accept text in via /v1/chat/completions and /v1/responses, not only the specialised /v1/realtime WebSocket transport. The previous regex hid them from the picker AND the new regression test pinned the regression -- both are fixed here. Removed `realtime` from the mid-id feature suffix set, matching the same reasoning that kept `audio` out. The audio-only endpoint families (`tts-*`, `whisper-*`, `*-transcribe`) still get caught above. Renamed the regression test to cover both families. --- studio/backend/core/inference/providers.py | 34 +++++++++--------- .../tests/test_provider_registry_filters.py | 35 ++++++++++++------- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 850566757f..018f3786ed 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -36,22 +36,21 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # # Pattern strategy: # - Unambiguous *feature* indicators (tts/whisper/transcribe/ - # realtime/image/embedding/moderation/sora) match anywhere - # in the id with `(?:^|-)` because OpenAI uses them as the - # primary qualifier on every variant -- e.g. canonical - # `tts-1` AND family variants `gpt-4o-tts`, `gpt-4o-mini-tts`. - # These words don't appear in legitimate chat model ids - # mid-string, so the mid-id match is intentional and safe. - # - `audio` is INTENTIONALLY NOT in the mid-id set: the - # `gpt-audio` / `gpt-audio-mini` / `gpt-audio-1.5` family is - # chat-completion-capable (text in / text or audio out via - # /v1/chat/completions and /v1/responses), so dropping them - # would hide supported chat models from the picker. The + # image/embedding/moderation/sora) match anywhere in the id + # with `(?:^|-)` because OpenAI uses them as the primary + # qualifier on every variant -- e.g. canonical `tts-1` AND + # family variants `gpt-4o-tts`, `gpt-4o-mini-tts`. These + # words don't appear in legitimate chat model ids mid-string, + # so the mid-id match is intentional and safe. + # - `audio` and `realtime` are INTENTIONALLY NOT in the + # mid-id set. Both the `gpt-audio*` and `gpt-realtime*` + # families are chat/responses-capable today -- per OpenAI's + # audio and realtime guides they accept text in via + # /v1/chat/completions and /v1/responses, not only the + # specialised /v1/realtime WebSocket transport. Dropping + # them hid supported chat models from the picker. The # audio-only endpoint families (`tts-*`, `whisper-*`, - # `*-transcribe`) ARE caught above. - # - `realtime` keeps the mid-id match because the - # `gpt-realtime*` family is a separate /v1/realtime - # (WebSocket) endpoint, not chat-completions. + # `*-transcribe`) are still caught above. # - `search` is INTENTIONALLY NOT in the mid-id set because # `gpt-4o-search-preview` and `gpt-4o-mini-search-preview` are # chat-with-retrieval models that absolutely belong in the @@ -69,9 +68,10 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # Verified against the live /v1/models listing 2026-05-22. "model_id_denylist": re.compile( # Feature suffixes that mark a non-chat variant on any base. - # Note: `audio` is omitted on purpose -- see docstring above. + # Note: `audio` and `realtime` are omitted on purpose -- + # see docstring above. r"(?:^|-)(?:embedding|tts|whisper|moderation|image|" - r"realtime|transcribe|sora)\b" + r"transcribe|sora)\b" # Legacy completion bases -- ^-anchored to avoid false # positives on hypothetical future chat ids containing # those words mid-string. diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index c055739c30..f8451e67ff 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -77,27 +77,39 @@ def test_openai_keeps_every_known_chat_family(): assert surviving == live, surviving -def test_openai_audio_family_is_chat_capable_and_kept(): +def test_openai_audio_and_realtime_families_are_chat_capable_and_kept(): # Pin the chat/non-chat split that lives in the regex comment: - # gpt-audio is kept (chat completions accept it), gpt-realtime is - # dropped (separate /v1/realtime endpoint), gpt-4o-transcribe is - # dropped (audio-only input via /v1/audio/transcriptions). + # gpt-audio AND gpt-realtime are kept (both accept text in via + # /v1/chat/completions and /v1/responses, not only their + # specialised transports); gpt-4o-transcribe etc. are dropped + # (audio-only input via /v1/audio/transcriptions). kept = _apply( "openai", [ "gpt-audio", "gpt-audio-1.5", "gpt-audio-mini", + "gpt-realtime", + "gpt-realtime-mini", + "gpt-realtime-1.5", + "gpt-realtime-2", + "gpt-4o-realtime-preview", ], ) - assert kept == ["gpt-audio", "gpt-audio-1.5", "gpt-audio-mini"], kept + assert kept == [ + "gpt-audio", + "gpt-audio-1.5", + "gpt-audio-mini", + "gpt-realtime", + "gpt-realtime-mini", + "gpt-realtime-1.5", + "gpt-realtime-2", + "gpt-4o-realtime-preview", + ], kept dropped = _apply( "openai", [ - "gpt-realtime", - "gpt-realtime-mini", - "gpt-4o-realtime-preview", "gpt-4o-transcribe", "gpt-4o-mini-transcribe", ], @@ -108,8 +120,9 @@ def test_openai_audio_family_is_chat_capable_and_kept(): def test_openai_drops_non_chat_ids(): noise = [ # Embeddings / TTS / image / moderation / whisper / etc. - # gpt-audio* is intentionally OMITTED from this list -- it is - # chat-capable. See test_openai_audio_family_is_chat_capable_and_kept. + # gpt-audio* and gpt-realtime* are intentionally OMITTED from + # this list -- both are chat-capable. See + # test_openai_audio_and_realtime_families_are_chat_capable_and_kept. "text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002", @@ -125,8 +138,6 @@ def test_openai_drops_non_chat_ids(): "gpt-image-2", "gpt-image-1-mini", "chatgpt-image-latest", - "gpt-realtime-2", - "gpt-4o-realtime-preview", "gpt-4o-transcribe", "gpt-4o-mini-transcribe", "gpt-4o-mini-tts", From 4104691b98b3ca729a6098af3e4084d9e31e06ac Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 12:48:06 +0000 Subject: [PATCH 06/13] Address review: scope OpenAI sampling drop to reasoning families only Codex P1: the denylist-based picker now admits non-reasoning chat families (gpt-4o*, gpt-4.1*, gpt-3.5-turbo*, gpt-audio*, gpt-realtime*) that ACCEPT temperature and top_p on /v1/responses. But _stream_openai_responses was unconditionally `del temperature, top_p` on the assumption that the old allowlist only let reasoning-class ids through. Users selecting newly-surfaced gpt-4o etc. saw their slider settings silently ignored, breaking reproducibility. Added an _OPENAI_REASONING_FAMILY regex covering gpt-5.x, o1/o3/o4, and gpt-4.5 (the only families that reject the sampling knobs). For non-reasoning models, forward temperature and top_p verbatim when the caller passes them; for reasoning models, keep dropping them with a comment pointing at reasoning.effort as the only accepted knob. Tests added: - test_responses_forwards_sampling_for_non_reasoning_chat_families: 8 models in (gpt-4o family, gpt-4.1, gpt-4, gpt-3.5-turbo, gpt-audio, gpt-realtime) all surface temperature=0.42 / top_p=0.85 on the outbound body. - test_responses_still_drops_sampling_for_reasoning_families: 7 reasoning models (gpt-5.5 / gpt-5.4 / gpt-5 / o1 / o3-mini / o4-mini / gpt-4.5-preview) still get both fields stripped. Existing gpt-5.5 test docstring updated to point at the sibling non-reasoning test instead of claiming the registry allowlist only admits reasoning families. --- .../core/inference/external_provider.py | 51 ++++++++-- .../test_openai_responses_translation.py | 98 ++++++++++++++++++- 2 files changed, 136 insertions(+), 13 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 25e1725337..781ef2c612 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -68,6 +68,23 @@ _ANTHROPIC_4_7_SAMPLING_REMOVED = re.compile( ) _OPENAI_REASONING_SUMMARY_UNSUPPORTED = re.compile(r"^o3(?:[-.]|$)") +# Reasoning-class OpenAI families that reject `temperature` and `top_p` +# with "Unsupported parameter" 400s on /v1/responses (and on +# /v1/chat/completions for the same families). Sampling knobs MUST be +# dropped for these. Other chat-completion families (gpt-4o*, +# gpt-4.1*, gpt-3.5-turbo*, gpt-4*, gpt-audio*, gpt-realtime*) +# accept the standard sampling shape and we forward both fields +# verbatim so the user's slider settings actually take effect. +# +# Source: OpenAI's reasoning guide +# (https://developers.openai.com/api/docs/guides/reasoning) and the +# /v1/responses reference. Covers gpt-5.x family, o1/o3/o4 family, +# and gpt-4.5 family. The `(?:[-.]|$)` anchor keeps gpt-50 / o30 +# hypotheticals out. +_OPENAI_REASONING_FAMILY = re.compile( + r"^(?:gpt-5(?:\.\d+)?|o[134]|gpt-4\.5)(?:[-.]|$)" +) + class _AnthropicThinkingSpec(NamedTuple): prefixes: tuple[str, ...] @@ -2650,21 +2667,35 @@ class ExternalProviderClient: if translated_parts: input_items.append({"role": role, "content": translated_parts}) - # NOTE: gpt-5.x / o3 / gpt-4.5 are reasoning-class models. They reject - # temperature and top_p with `Unsupported parameter` 400s on - # /v1/responses (and on /v1/chat/completions for the same families). - # The PROVIDER_REGISTRY['openai'] model_id_allowlist already scopes - # the picker to those families, so we never need to send sampling - # knobs here. ``reasoning.effort`` defaults to "medium" server-side - # if omitted — surface it in a future commit if a knob is wanted. - del temperature, top_p # explicit drop — params are accepted for - # API symmetry with the other stream methods but not forwarded. - + # Sampling knob handling. The old allowlist-scoped picker only + # admitted reasoning-class families (gpt-5.x / o3 / gpt-4.5), + # all of which reject `temperature` and `top_p` with + # "Unsupported parameter" 400s on /v1/responses -- so the + # earlier code unconditionally dropped both. + # + # The denylist-based picker (PR 5684) now also admits + # non-reasoning chat models (gpt-4o*, gpt-4.1*, gpt-4*, + # gpt-3.5-turbo*, gpt-audio*, gpt-realtime*) that accept the + # standard sampling shape. Forward temperature/top_p for those; + # keep dropping them for the reasoning families. body: dict[str, Any] = { "model": model, "input": input_items, "stream": True, } + is_reasoning_family = bool( + _OPENAI_REASONING_FAMILY.match(model.strip().lower()) + ) + if not is_reasoning_family: + if temperature is not None: + body["temperature"] = temperature + if top_p is not None: + body["top_p"] = top_p + else: + # Reasoning-class: explicit drop. ``reasoning.effort`` is + # the only knob the API accepts and defaults to "medium" + # server-side if omitted. + del temperature, top_p # `summary: "auto"` is what makes /v1/responses emit reasoning # summary events — without it OpenAI returns no thinking text on # most reasoning models, the SSE handler has no diff --git a/studio/backend/tests/test_openai_responses_translation.py b/studio/backend/tests/test_openai_responses_translation.py index 22ccba7058..51bb3c2343 100644 --- a/studio/backend/tests/test_openai_responses_translation.py +++ b/studio/backend/tests/test_openai_responses_translation.py @@ -101,9 +101,11 @@ def test_responses_request_body_uses_input_and_instructions(monkeypatch): assert body["input"] == [{"role": "user", "content": "Hi"}] assert body["max_output_tokens"] == 512 assert body["stream"] is True - # Responses API on reasoning-class models (gpt-5.x / o3 / gpt-4.5 — the - # only OpenAI ids the registry allowlist exposes) rejects these as - # `Unsupported parameter`. Make sure we never silently forward them. + # gpt-5.5 is reasoning-class: Responses API rejects temperature + # and top_p with `Unsupported parameter` 400s, so the helper must + # drop them. Non-reasoning models (gpt-4o*, gpt-4.1*, gpt-3.5-turbo*, + # gpt-audio*, gpt-realtime*) are covered in a sibling test that + # asserts the inverse forwarding. assert "temperature" not in body assert "top_p" not in body assert "presence_penalty" not in body @@ -112,6 +114,96 @@ def test_responses_request_body_uses_input_and_instructions(monkeypatch): assert "messages" not in body +def test_responses_forwards_sampling_for_non_reasoning_chat_families(monkeypatch): + # Codex P1 follow-up to the picker filter change (PR 5684): the + # OpenAI denylist now admits non-reasoning chat families + # (gpt-4o*, gpt-4.1*, gpt-3.5-turbo*, gpt-audio*, gpt-realtime*). + # The Responses API ACCEPTS temperature and top_p on those, so + # the helper must forward the user's slider settings instead of + # silently dropping them. + for model in ( + "gpt-4o", + "gpt-4o-mini", + "gpt-4o-2026-01-01", + "gpt-4.1", + "gpt-4", + "gpt-3.5-turbo", + "gpt-audio", + "gpt-realtime", + ): + captured: dict = {} + + def handler(request: httpx.Request, _captured = captured) -> httpx.Response: + _captured["body"] = json.loads(request.content.decode("utf-8")) + return httpx.Response( + 200, + content = _responses_sse( + [{"type": "response.completed", "response": {}}] + ), + headers = {"content-type": "text/event-stream"}, + ) + + _mock_http_client(monkeypatch, handler) + + async def run(): + client = _make_client() + async for _ in client._stream_openai_responses( + messages = [{"role": "user", "content": "Hi"}], + model = model, + temperature = 0.42, + top_p = 0.85, + max_tokens = 32, + enable_thinking = None, + reasoning_effort = None, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert body["temperature"] == 0.42, (model, body) + assert body["top_p"] == 0.85, (model, body) + + +def test_responses_still_drops_sampling_for_reasoning_families(monkeypatch): + # Sanity: parametrise across the reasoning-class families so a + # future regex tweak that accidentally weakens the drop surfaces + # here. + for model in ("gpt-5.5", "gpt-5.4", "gpt-5", "o1", "o3-mini", "o4-mini", "gpt-4.5-preview"): + captured: dict = {} + + def handler(request: httpx.Request, _captured = captured) -> httpx.Response: + _captured["body"] = json.loads(request.content.decode("utf-8")) + return httpx.Response( + 200, + content = _responses_sse( + [{"type": "response.completed", "response": {}}] + ), + headers = {"content-type": "text/event-stream"}, + ) + + _mock_http_client(monkeypatch, handler) + + async def run(): + client = _make_client() + async for _ in client._stream_openai_responses( + messages = [{"role": "user", "content": "Hi"}], + model = model, + temperature = 0.42, + top_p = 0.85, + max_tokens = 32, + enable_thinking = None, + reasoning_effort = None, + ): + pass + await client.close() + + _drive(run()) + body = captured["body"] + assert "temperature" not in body, (model, body) + assert "top_p" not in body, (model, body) + + def test_responses_translates_image_parts(monkeypatch): captured: dict = {} From 27732404a36f5e1c81ead2210f37ebfd44f36d9c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 22 May 2026 12:48:18 +0000 Subject: [PATCH 07/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/external_provider.py | 4 +--- .../backend/tests/test_openai_responses_translation.py | 10 +++++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 781ef2c612..b63c1f818c 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -81,9 +81,7 @@ _OPENAI_REASONING_SUMMARY_UNSUPPORTED = re.compile(r"^o3(?:[-.]|$)") # /v1/responses reference. Covers gpt-5.x family, o1/o3/o4 family, # and gpt-4.5 family. The `(?:[-.]|$)` anchor keeps gpt-50 / o30 # hypotheticals out. -_OPENAI_REASONING_FAMILY = re.compile( - r"^(?:gpt-5(?:\.\d+)?|o[134]|gpt-4\.5)(?:[-.]|$)" -) +_OPENAI_REASONING_FAMILY = re.compile(r"^(?:gpt-5(?:\.\d+)?|o[134]|gpt-4\.5)(?:[-.]|$)") class _AnthropicThinkingSpec(NamedTuple): diff --git a/studio/backend/tests/test_openai_responses_translation.py b/studio/backend/tests/test_openai_responses_translation.py index 51bb3c2343..390a6926f1 100644 --- a/studio/backend/tests/test_openai_responses_translation.py +++ b/studio/backend/tests/test_openai_responses_translation.py @@ -169,7 +169,15 @@ def test_responses_still_drops_sampling_for_reasoning_families(monkeypatch): # Sanity: parametrise across the reasoning-class families so a # future regex tweak that accidentally weakens the drop surfaces # here. - for model in ("gpt-5.5", "gpt-5.4", "gpt-5", "o1", "o3-mini", "o4-mini", "gpt-4.5-preview"): + for model in ( + "gpt-5.5", + "gpt-5.4", + "gpt-5", + "o1", + "o3-mini", + "o4-mini", + "gpt-4.5-preview", + ): captured: dict = {} def handler(request: httpx.Request, _captured = captured) -> httpx.Response: From 3661243e350a573132cf7be7b05e4b15c42bdc43 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Fri, 22 May 2026 17:33:24 +0000 Subject: [PATCH 08/13] Address review: deny realtime-translate and *-instruct in OpenAI filter Codex flagged two leaks in the new denylist: * `gpt-realtime-translate*` / `gpt-audio-translate*` are audio translation variants. They share the gpt-realtime/gpt-audio name root we now intentionally keep, but the chat picker only knows the chat/responses transport and they would 4xx on selection. * `gpt-3.5-turbo-instruct*` (and any future legacy id ending in -instruct) speaks /v1/completions, not chat/responses. The previous denylist let them through because nothing in the pattern matched. Extend the feature-suffix alternation to cover `translate` and `instruct`. New regression tests pin both exclusions plus the fact that bare gpt-realtime / gpt-realtime-mini / gpt-audio still survive. --- studio/backend/core/inference/providers.py | 11 +++- .../tests/test_provider_registry_filters.py | 50 +++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 018f3786ed..166f4abf63 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -69,9 +69,16 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { "model_id_denylist": re.compile( # Feature suffixes that mark a non-chat variant on any base. # Note: `audio` and `realtime` are omitted on purpose -- - # see docstring above. + # see docstring above. `translate` is here so the audio + # translation variants `gpt-realtime-translate*` / + # `gpt-audio-translate*` are dropped even though we keep + # the parent `gpt-realtime` / `gpt-audio` chat families. + # `instruct` catches the legacy completion-only suffix + # used by `gpt-3.5-turbo-instruct*`; that family is wired + # to /v1/completions and would 4xx on our chat/responses + # transport, so it should not surface in the chat picker. r"(?:^|-)(?:embedding|tts|whisper|moderation|image|" - r"transcribe|sora)\b" + r"transcribe|translate|instruct|sora)\b" # Legacy completion bases -- ^-anchored to avoid false # positives on hypothetical future chat ids containing # those words mid-string. diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index f8451e67ff..d366764a5f 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -168,6 +168,56 @@ def test_openai_drops_non_chat_ids(): assert surviving == [], surviving +def test_openai_realtime_translate_variants_are_dropped_but_parent_chat_family_survives(): + """gpt-realtime / gpt-realtime-mini are chat-capable and stay, + but the audio-translation variants share the chat picker's + transport and would 4xx, so they must be filtered. Pin both + behaviours so a future regex tweak cannot collapse one into + the other.""" + kept = _apply( + "openai", + [ + "gpt-realtime", + "gpt-realtime-mini", + "gpt-audio", + "gpt-audio-mini", + ], + ) + assert kept == [ + "gpt-realtime", + "gpt-realtime-mini", + "gpt-audio", + "gpt-audio-mini", + ], kept + + dropped = _apply( + "openai", + [ + "gpt-realtime-translate", + "gpt-realtime-translate-mini", + "gpt-audio-translate", + "gpt-4o-realtime-translate-preview", + ], + ) + assert dropped == [], dropped + + +def test_openai_legacy_instruct_completion_ids_are_dropped(): + """Legacy `*-instruct` completion-only ids (gpt-3.5-turbo-instruct + and friends) speak /v1/completions, not chat/responses. Our OpenAI + bridge only knows the chat/responses transport, so admitting them + into the picker would 4xx every selection.""" + dropped = _apply( + "openai", + [ + "gpt-3.5-turbo-instruct", + "gpt-3.5-turbo-instruct-0914", + "davinci-002-instruct", + ], + ) + assert dropped == [], dropped + + def test_openai_search_preview_is_kept_search_api_is_dropped(): # Pin the search-vs-search-api distinction so a future regex tweak # doesn't silently regress to dropping chat-with-search models. From 1a7238c753030d8fb745090ea503add46b998703 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 16:33:32 +0000 Subject: [PATCH 09/13] Studio: drop non-streaming gpt-realtime + gpt-audio-mini from picker Codex P1 round 2 on this PR: by allowing the full gpt-realtime and gpt-audio families through the denylist we let the chat picker surface IDs that OpenAI's model cards mark Streaming: Not supported (gpt- realtime, gpt-realtime-mini, gpt-audio-mini -- Realtime-API/WebSocket only). Studio's path always sends stream:true, so selecting them 4xx'd at request time instead of producing a reply. Narrow the regex to drop ^gpt-realtime(?:$|-) and ^gpt-audio-mini\b; gpt-audio (chat-streams over /v1/chat/completions) and gpt-4o-realtime- preview (chat-streams via the /v1/responses adapter) still pass. Updated test_provider_registry_filters.py to pin the new chat/non-chat split: 9/9 filter tests pass. --- studio/backend/core/inference/providers.py | 20 +++--- .../tests/test_provider_registry_filters.py | 69 +++++++------------ 2 files changed, 36 insertions(+), 53 deletions(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 166f4abf63..7ff800e7e9 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -68,17 +68,19 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # Verified against the live /v1/models listing 2026-05-22. "model_id_denylist": re.compile( # Feature suffixes that mark a non-chat variant on any base. - # Note: `audio` and `realtime` are omitted on purpose -- - # see docstring above. `translate` is here so the audio - # translation variants `gpt-realtime-translate*` / - # `gpt-audio-translate*` are dropped even though we keep - # the parent `gpt-realtime` / `gpt-audio` chat families. - # `instruct` catches the legacy completion-only suffix - # used by `gpt-3.5-turbo-instruct*`; that family is wired - # to /v1/completions and would 4xx on our chat/responses - # transport, so it should not surface in the chat picker. + # `audio`/`realtime` are NOT in this group so `gpt-audio` + # (chat-capable, streaming) stays; specific non-streaming + # IDs are caught further down. r"(?:^|-)(?:embedding|tts|whisper|moderation|image|" r"transcribe|translate|instruct|sora)\b" + # OpenAI realtime/audio variants that don't support chat + # streaming. Studio always sends `stream: true`, and OpenAI + # marks `gpt-realtime*` and `gpt-audio-mini` as Streaming: + # Not supported (Realtime API only); surfacing them in the + # picker would 4xx at request time. `gpt-audio` and + # `gpt-4o-realtime-preview` still stream and are kept. + r"|^gpt-realtime(?:$|-)" + r"|^gpt-audio-mini\b" # Legacy completion bases -- ^-anchored to avoid false # positives on hypothetical future chat ids containing # those words mid-string. diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index d366764a5f..3aa9b23b2e 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -56,12 +56,10 @@ def test_openai_keeps_every_known_chat_family(): # search-api suffix. "gpt-4o-search-preview", "gpt-4o-mini-search-preview", - # gpt-audio family: chat-completion-capable (text in / text or - # audio out) per OpenAI's audio guide. Earlier revisions of - # this denylist dropped them via `(?:^|-)audio`; the regex - # comment block now keeps them. + # gpt-audio is chat-completion-capable with streaming. The + # `-mini` variant does NOT stream (Realtime API only) and is + # asserted dropped in the realtime/audio split test below. "gpt-audio", - "gpt-audio-mini", "gpt-audio-1.5", # Hypothetical future families that the old allowlist would have # silently dropped -- they MUST surface under the new denylist. @@ -77,39 +75,35 @@ def test_openai_keeps_every_known_chat_family(): assert surviving == live, surviving -def test_openai_audio_and_realtime_families_are_chat_capable_and_kept(): - # Pin the chat/non-chat split that lives in the regex comment: - # gpt-audio AND gpt-realtime are kept (both accept text in via - # /v1/chat/completions and /v1/responses, not only their - # specialised transports); gpt-4o-transcribe etc. are dropped - # (audio-only input via /v1/audio/transcriptions). +def test_openai_audio_and_realtime_families_split_on_streaming_support(): + # Pin the chat/non-chat split: Studio always sends `stream: true`, + # so the picker keeps gpt-audio* (chat streams over /v1/chat/ + # completions) and gpt-4o-realtime-preview (chat streams via the + # /v1/responses adapter), but drops the new gpt-realtime* family + # and gpt-audio-mini, which OpenAI's model cards mark Streaming: + # Not supported (Realtime API / WebSocket only). kept = _apply( "openai", [ "gpt-audio", "gpt-audio-1.5", - "gpt-audio-mini", - "gpt-realtime", - "gpt-realtime-mini", - "gpt-realtime-1.5", - "gpt-realtime-2", "gpt-4o-realtime-preview", ], ) assert kept == [ "gpt-audio", "gpt-audio-1.5", - "gpt-audio-mini", - "gpt-realtime", - "gpt-realtime-mini", - "gpt-realtime-1.5", - "gpt-realtime-2", "gpt-4o-realtime-preview", ], kept dropped = _apply( "openai", [ + "gpt-audio-mini", + "gpt-realtime", + "gpt-realtime-mini", + "gpt-realtime-1.5", + "gpt-realtime-2", "gpt-4o-transcribe", "gpt-4o-mini-transcribe", ], @@ -120,9 +114,9 @@ def test_openai_audio_and_realtime_families_are_chat_capable_and_kept(): def test_openai_drops_non_chat_ids(): noise = [ # Embeddings / TTS / image / moderation / whisper / etc. - # gpt-audio* and gpt-realtime* are intentionally OMITTED from - # this list -- both are chat-capable. See - # test_openai_audio_and_realtime_families_are_chat_capable_and_kept. + # gpt-audio (no -mini) and gpt-4o-realtime-preview are + # intentionally OMITTED from this list -- they DO stream chat. + # See test_openai_audio_and_realtime_families_split_on_streaming_support. "text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002", @@ -169,26 +163,13 @@ def test_openai_drops_non_chat_ids(): def test_openai_realtime_translate_variants_are_dropped_but_parent_chat_family_survives(): - """gpt-realtime / gpt-realtime-mini are chat-capable and stay, - but the audio-translation variants share the chat picker's - transport and would 4xx, so they must be filtered. Pin both - behaviours so a future regex tweak cannot collapse one into - the other.""" - kept = _apply( - "openai", - [ - "gpt-realtime", - "gpt-realtime-mini", - "gpt-audio", - "gpt-audio-mini", - ], - ) - assert kept == [ - "gpt-realtime", - "gpt-realtime-mini", - "gpt-audio", - "gpt-audio-mini", - ], kept + """gpt-audio is chat-capable (streams over /v1/chat/completions) + and stays, but the audio-translation variants share the chat + picker's transport and would 4xx, so they must be filtered. The + new gpt-realtime* family and gpt-audio-mini are Realtime-only and + are dropped by a sibling assertion.""" + kept = _apply("openai", ["gpt-audio"]) + assert kept == ["gpt-audio"], kept dropped = _apply( "openai", From e457a493c266a957237ce46229399c7fcd60f4e2 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 16:49:54 +0000 Subject: [PATCH 10/13] fix(openai): also drop gpt-4o-realtime-preview from picker Codex follow-up P1 on PR #5684 (providers.py:83): the prior round kept `gpt-4o-realtime-preview` because I'd assumed it streams via the /v1/responses adapter, but OpenAI's own model card and the Realtime API docs confirm it is Realtime-only (WebRTC / WebSocket transport). Studio always sends `stream: true` over Chat Completions / Responses, so surfacing the model from the picker would 4xx at request time. Extends the denylist with `^gpt-4o(?:-mini)?-realtime\b`, covering `gpt-4o-realtime-preview` and `gpt-4o-mini-realtime-preview` (and any future dated/suffixed variants). The audio-chat families (`gpt-audio`, `gpt-audio-1.5`, `gpt-4o-audio-preview`) still stream over Chat Completions and are explicitly kept. `gpt-audio-mini` remains denied (Realtime-only per OpenAI). Updated `test_openai_audio_and_realtime_families_split_on_streaming_support` to pin the new split: `gpt-4o-audio-preview` is the surviving audio chat id, and `gpt-4o-realtime-preview` / `gpt-4o-mini-realtime-preview` join the dropped Realtime cohort. All 9 filter tests pass locally. --- studio/backend/core/inference/providers.py | 12 ++++--- .../tests/test_provider_registry_filters.py | 32 ++++++++++++------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 7ff800e7e9..9ec1db3e6d 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -75,11 +75,15 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { r"transcribe|translate|instruct|sora)\b" # OpenAI realtime/audio variants that don't support chat # streaming. Studio always sends `stream: true`, and OpenAI - # marks `gpt-realtime*` and `gpt-audio-mini` as Streaming: - # Not supported (Realtime API only); surfacing them in the - # picker would 4xx at request time. `gpt-audio` and - # `gpt-4o-realtime-preview` still stream and are kept. + # marks the full Realtime family as Streaming: Not supported + # (Realtime API only, WebRTC/WebSocket transport): `gpt-realtime`, + # `gpt-realtime-mini`, `gpt-4o-realtime-preview*` and + # `gpt-4o-mini-realtime-preview*`, plus `gpt-audio-mini`. + # Selecting them from the picker would 4xx at request time. + # `gpt-audio` and `gpt-4o-audio-preview` are kept -- they + # stream over Chat Completions / Responses. r"|^gpt-realtime(?:$|-)" + r"|^gpt-4o(?:-mini)?-realtime\b" r"|^gpt-audio-mini\b" # Legacy completion bases -- ^-anchored to avoid false # positives on hypothetical future chat ids containing diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index 3aa9b23b2e..e061fffac9 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -77,23 +77,28 @@ def test_openai_keeps_every_known_chat_family(): def test_openai_audio_and_realtime_families_split_on_streaming_support(): # Pin the chat/non-chat split: Studio always sends `stream: true`, - # so the picker keeps gpt-audio* (chat streams over /v1/chat/ - # completions) and gpt-4o-realtime-preview (chat streams via the - # /v1/responses adapter), but drops the new gpt-realtime* family - # and gpt-audio-mini, which OpenAI's model cards mark Streaming: - # Not supported (Realtime API / WebSocket only). + # so the picker keeps the audio chat families (gpt-audio* without + # -mini, plus gpt-4o-audio-preview*) which stream over + # /v1/chat/completions and /v1/responses, but drops the entire + # Realtime family (gpt-realtime*, gpt-4o-realtime-preview*, + # gpt-4o-mini-realtime-preview*) and gpt-audio-mini, which OpenAI's + # model cards mark Streaming: Not supported (Realtime API only, + # WebRTC/WebSocket transport). kept = _apply( "openai", [ "gpt-audio", "gpt-audio-1.5", - "gpt-4o-realtime-preview", + "gpt-4o-audio-preview", + "gpt-4o-audio-preview-2024-12-17", ], ) assert kept == [ "gpt-audio", "gpt-audio-1.5", - "gpt-4o-realtime-preview", + "gpt-4o-audio-preview", + # Dated audio-preview snapshot is dropped by the date-suffix rule; + # the canonical id above survives. ], kept dropped = _apply( @@ -104,6 +109,8 @@ def test_openai_audio_and_realtime_families_split_on_streaming_support(): "gpt-realtime-mini", "gpt-realtime-1.5", "gpt-realtime-2", + "gpt-4o-realtime-preview", + "gpt-4o-mini-realtime-preview", "gpt-4o-transcribe", "gpt-4o-mini-transcribe", ], @@ -114,9 +121,9 @@ def test_openai_audio_and_realtime_families_split_on_streaming_support(): def test_openai_drops_non_chat_ids(): noise = [ # Embeddings / TTS / image / moderation / whisper / etc. - # gpt-audio (no -mini) and gpt-4o-realtime-preview are - # intentionally OMITTED from this list -- they DO stream chat. - # See test_openai_audio_and_realtime_families_split_on_streaming_support. + # gpt-audio (no -mini) and gpt-4o-audio-preview are intentionally + # OMITTED from this list -- they DO stream chat. See + # test_openai_audio_and_realtime_families_split_on_streaming_support. "text-embedding-3-small", "text-embedding-3-large", "text-embedding-ada-002", @@ -166,8 +173,9 @@ def test_openai_realtime_translate_variants_are_dropped_but_parent_chat_family_s """gpt-audio is chat-capable (streams over /v1/chat/completions) and stays, but the audio-translation variants share the chat picker's transport and would 4xx, so they must be filtered. The - new gpt-realtime* family and gpt-audio-mini are Realtime-only and - are dropped by a sibling assertion.""" + full Realtime family (gpt-realtime*, gpt-4o*-realtime-preview*) + plus gpt-audio-mini are Realtime-only and dropped by a sibling + assertion.""" kept = _apply("openai", ["gpt-audio"]) assert kept == ["gpt-audio"], kept From d8e8cbb38a3d8cbb057680443cea340069f7de01 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 17:31:20 +0000 Subject: [PATCH 11/13] ci: re-trigger after transient Upload Playwright artifacts failure From 2efc67761d92873a7b222711fcaa65f7bb0c5335 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 18:31:04 +0000 Subject: [PATCH 12/13] ci: re-trigger after transient HTTP error fetching ggml-org/llama.cpp release metadata From f5ace8824cf2aacf59abc673a003a6d5e3a2972f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 23 May 2026 18:33:27 +0000 Subject: [PATCH 13/13] fix(openai): drop legacy MMDD snapshot suffixes from picker Codex P2 follow-up on PR #5684 (providers.py:104): the new OpenAI denylist only stripped the modern `-YYYY-MM-DD` dated form, so legacy compact snapshots like `gpt-3.5-turbo-0125`, `gpt-4-0613`, `gpt-4-1106-preview`, and `gpt-4-0125-preview` slipped through. The listing returns both the snapshot and the canonical id; the picker should surface only the canonical so users do not accidentally select a deprecated snapshot. Extends the dated-suffix branch with `-\d{4}(?:-preview)?$` to cover the compact `MMDD` form and the `-MMDD-preview` variant. The new rule is anchored to the end of the id so canonical chat ids whose minor version happens to be a year-like number (`gpt-4.5`, `gpt-5.5`, `o3`) are unaffected. Added `test_openai_legacy_compact_snapshot_suffixes_are_dropped` to pin both sides (8 snapshot ids dropped, 7 canonical ids kept). 10 / 10 filter tests pass locally. --- studio/backend/core/inference/providers.py | 8 +++- .../tests/test_provider_registry_filters.py | 47 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 9ec1db3e6d..82111e9c90 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -100,8 +100,14 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # Fine-tunes carry the user's tenant in the id. r"|^ft:" # Dated snapshot suffixes hide behind the canonical id - # which is also in the listing. + # which is also in the listing. Covers both the modern + # `-YYYY-MM-DD` dated form and the legacy compact + # 4-digit `MMDD` form (e.g. `gpt-3.5-turbo-0125`, + # `gpt-4-0613`, `gpt-4-1106-preview`) so the picker only + # surfaces the canonical id even when the listing still + # returns the snapshot copy. r"|-\d{4}-\d{2}-\d{2}$" + r"|-\d{4}(?:-preview)?$" ), }, "anthropic": { diff --git a/studio/backend/tests/test_provider_registry_filters.py b/studio/backend/tests/test_provider_registry_filters.py index e061fffac9..bd766261dd 100644 --- a/studio/backend/tests/test_provider_registry_filters.py +++ b/studio/backend/tests/test_provider_registry_filters.py @@ -207,6 +207,53 @@ def test_openai_legacy_instruct_completion_ids_are_dropped(): assert dropped == [], dropped +def test_openai_legacy_compact_snapshot_suffixes_are_dropped(): + """Legacy `-MMDD` snapshot suffixes (gpt-3.5-turbo-0125, + gpt-4-0613, gpt-4-1106-preview, etc.) hide behind the canonical + id which the listing also returns; surface only the canonical so + users do not pick a deprecated snapshot by accident. The + `-\\d{4}(?:-preview)?$` rule must not catch canonical ids whose + minor version happens to be a year-like number (e.g. gpt-4.5, + o3) -- those are tested as KEEP below.""" + dropped = _apply( + "openai", + [ + "gpt-3.5-turbo-0125", + "gpt-3.5-turbo-0301", + "gpt-3.5-turbo-16k-0613", + "gpt-4-0613", + "gpt-4-0314", + "gpt-4-32k-0613", + "gpt-4-1106-preview", + "gpt-4-0125-preview", + ], + ) + assert dropped == [], dropped + + # Canonical chat ids that share a digit-heavy tail must survive. + kept = _apply( + "openai", + [ + "gpt-3.5-turbo", + "gpt-4o", + "gpt-4.5", + "gpt-5.5", + "gpt-5.5-mini", + "gpt-5.5-pro", + "o3", + ], + ) + assert set(kept) >= { + "gpt-3.5-turbo", + "gpt-4o", + "gpt-4.5", + "gpt-5.5", + "gpt-5.5-mini", + "gpt-5.5-pro", + "o3", + }, kept + + def test_openai_search_preview_is_kept_search_api_is_dropped(): # Pin the search-vs-search-api distinction so a future regex tweak # doesn't silently regress to dropping chat-with-search models.