Compare commits
13 commits
main
...
feat/model
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f5ace8824c | ||
|
|
2efc67761d | ||
|
|
d8e8cbb38a | ||
|
|
e457a493c2 | ||
|
|
1a7238c753 | ||
|
|
3661243e35 | ||
|
|
27732404a3 | ||
|
|
4104691b98 | ||
|
|
6ea20c5aa7 | ||
|
|
c51526d244 | ||
|
|
9a9c6b0949 | ||
|
|
4434e1eeb7 | ||
|
|
1c21d1d3f4 |
4 changed files with 567 additions and 30 deletions
|
|
@ -68,6 +68,21 @@ _ANTHROPIC_4_7_SAMPLING_REMOVED = re.compile(
|
|||
)
|
||||
_OPENAI_REASONING_SUMMARY_UNSUPPORTED = re.compile(r"^o3(?:[-.]|$)")
|
||||
|
||||
# Reasoning-class OpenAI families that reject `temperature` and `top_p`
|
||||
# with "Unsupported parameter" 400s on /v1/responses (and on
|
||||
# /v1/chat/completions for the same families). Sampling knobs MUST be
|
||||
# dropped for these. Other chat-completion families (gpt-4o*,
|
||||
# gpt-4.1*, gpt-3.5-turbo*, gpt-4*, gpt-audio*, gpt-realtime*)
|
||||
# accept the standard sampling shape and we forward both fields
|
||||
# verbatim so the user's slider settings actually take effect.
|
||||
#
|
||||
# Source: OpenAI's reasoning guide
|
||||
# (https://developers.openai.com/api/docs/guides/reasoning) and the
|
||||
# /v1/responses reference. Covers gpt-5.x family, o1/o3/o4 family,
|
||||
# and gpt-4.5 family. The `(?:[-.]|$)` anchor keeps gpt-50 / o30
|
||||
# hypotheticals out.
|
||||
_OPENAI_REASONING_FAMILY = re.compile(r"^(?:gpt-5(?:\.\d+)?|o[134]|gpt-4\.5)(?:[-.]|$)")
|
||||
|
||||
|
||||
class _AnthropicThinkingSpec(NamedTuple):
|
||||
prefixes: tuple[str, ...]
|
||||
|
|
@ -2650,21 +2665,35 @@ class ExternalProviderClient:
|
|||
if translated_parts:
|
||||
input_items.append({"role": role, "content": translated_parts})
|
||||
|
||||
# NOTE: gpt-5.x / o3 / gpt-4.5 are reasoning-class models. They reject
|
||||
# temperature and top_p with `Unsupported parameter` 400s on
|
||||
# /v1/responses (and on /v1/chat/completions for the same families).
|
||||
# The PROVIDER_REGISTRY['openai'] model_id_allowlist already scopes
|
||||
# the picker to those families, so we never need to send sampling
|
||||
# knobs here. ``reasoning.effort`` defaults to "medium" server-side
|
||||
# if omitted — surface it in a future commit if a knob is wanted.
|
||||
del temperature, top_p # explicit drop — params are accepted for
|
||||
# API symmetry with the other stream methods but not forwarded.
|
||||
|
||||
# Sampling knob handling. The old allowlist-scoped picker only
|
||||
# admitted reasoning-class families (gpt-5.x / o3 / gpt-4.5),
|
||||
# all of which reject `temperature` and `top_p` with
|
||||
# "Unsupported parameter" 400s on /v1/responses -- so the
|
||||
# earlier code unconditionally dropped both.
|
||||
#
|
||||
# The denylist-based picker (PR 5684) now also admits
|
||||
# non-reasoning chat models (gpt-4o*, gpt-4.1*, gpt-4*,
|
||||
# gpt-3.5-turbo*, gpt-audio*, gpt-realtime*) that accept the
|
||||
# standard sampling shape. Forward temperature/top_p for those;
|
||||
# keep dropping them for the reasoning families.
|
||||
body: dict[str, Any] = {
|
||||
"model": model,
|
||||
"input": input_items,
|
||||
"stream": True,
|
||||
}
|
||||
is_reasoning_family = bool(
|
||||
_OPENAI_REASONING_FAMILY.match(model.strip().lower())
|
||||
)
|
||||
if not is_reasoning_family:
|
||||
if temperature is not None:
|
||||
body["temperature"] = temperature
|
||||
if top_p is not None:
|
||||
body["top_p"] = top_p
|
||||
else:
|
||||
# Reasoning-class: explicit drop. ``reasoning.effort`` is
|
||||
# the only knob the API accepts and defaults to "medium"
|
||||
# server-side if omitted.
|
||||
del temperature, top_p
|
||||
# `summary: "auto"` is what makes /v1/responses emit reasoning
|
||||
# summary events — without it OpenAI returns no thinking text on
|
||||
# most reasoning models, the SSE handler has no <think>…</think>
|
||||
|
|
|
|||
|
|
@ -26,14 +26,89 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
|
|||
"supports_tool_calling": True,
|
||||
"auth_header": "Authorization",
|
||||
"auth_prefix": "Bearer ",
|
||||
# Keep the model picker scoped to the current generation. The remote
|
||||
# /v1/models listing returns dozens of historical snapshots, fine-tunes
|
||||
# and non-chat models (embeddings, TTS, image, moderation) that we
|
||||
# never want to surface in the chat UI. Filtering here so backend
|
||||
# is the single source of truth.
|
||||
"model_id_allowlist": re.compile(r"^(gpt-5\.[345]|gpt-4\.5|o3)(?:[-.]|$)"),
|
||||
# Hide dated snapshots and the retired plain gpt-5.3 id.
|
||||
"model_id_denylist": re.compile(r"^(gpt-5\.3)$|-\d{4}-\d{2}-\d{2}$"),
|
||||
# The remote /v1/models listing returns the full account catalog,
|
||||
# including non-chat models (embeddings, TTS, image, moderation,
|
||||
# whisper, dall-e) and fine-tunes. Previously we used a hardcoded
|
||||
# family allowlist (`gpt-5\.[345]|gpt-4\.5|o3`) which silently
|
||||
# dropped every new family OpenAI shipped, including the gpt-5.5
|
||||
# generation today. Switch to a non-chat denylist instead so any
|
||||
# new chat family auto-appears the moment OpenAI lists it.
|
||||
#
|
||||
# Pattern strategy:
|
||||
# - Unambiguous *feature* indicators (tts/whisper/transcribe/
|
||||
# image/embedding/moderation/sora) match anywhere in the id
|
||||
# with `(?:^|-)` because OpenAI uses them as the primary
|
||||
# qualifier on every variant -- e.g. canonical `tts-1` AND
|
||||
# family variants `gpt-4o-tts`, `gpt-4o-mini-tts`. These
|
||||
# words don't appear in legitimate chat model ids mid-string,
|
||||
# so the mid-id match is intentional and safe.
|
||||
# - `audio` and `realtime` are INTENTIONALLY NOT in the
|
||||
# mid-id set. Both the `gpt-audio*` and `gpt-realtime*`
|
||||
# families are chat/responses-capable today -- per OpenAI's
|
||||
# audio and realtime guides they accept text in via
|
||||
# /v1/chat/completions and /v1/responses, not only the
|
||||
# specialised /v1/realtime WebSocket transport. Dropping
|
||||
# them hid supported chat models from the picker. The
|
||||
# audio-only endpoint families (`tts-*`, `whisper-*`,
|
||||
# `*-transcribe`) are still caught above.
|
||||
# - `search` is INTENTIONALLY NOT in the mid-id set because
|
||||
# `gpt-4o-search-preview` and `gpt-4o-mini-search-preview` are
|
||||
# chat-with-retrieval models that absolutely belong in the
|
||||
# picker. The standalone search API is caught separately by
|
||||
# `-search-api` (suffix) which never appears on a chat id.
|
||||
# - Legacy completion bases (babbage / davinci / ada / curie)
|
||||
# are ^-anchored: they ONLY ever begin a legacy id like
|
||||
# `babbage-002`, `davinci-002`, `text-davinci-003`. A future
|
||||
# hypothetical `gpt-7-davinci-edition` chat model would NOT
|
||||
# be dropped, which is the right default for a denylist.
|
||||
# - Prefix-only patterns (`^dall-e`, `^computer-use`, `^ft:`,
|
||||
# `^gpt-image`) cover canonical non-chat families without
|
||||
# risk of mid-id false positives.
|
||||
#
|
||||
# Verified against the live /v1/models listing 2026-05-22.
|
||||
"model_id_denylist": re.compile(
|
||||
# Feature suffixes that mark a non-chat variant on any base.
|
||||
# `audio`/`realtime` are NOT in this group so `gpt-audio`
|
||||
# (chat-capable, streaming) stays; specific non-streaming
|
||||
# IDs are caught further down.
|
||||
r"(?:^|-)(?:embedding|tts|whisper|moderation|image|"
|
||||
r"transcribe|translate|instruct|sora)\b"
|
||||
# OpenAI realtime/audio variants that don't support chat
|
||||
# streaming. Studio always sends `stream: true`, and OpenAI
|
||||
# marks the full Realtime family as Streaming: Not supported
|
||||
# (Realtime API only, WebRTC/WebSocket transport): `gpt-realtime`,
|
||||
# `gpt-realtime-mini`, `gpt-4o-realtime-preview*` and
|
||||
# `gpt-4o-mini-realtime-preview*`, plus `gpt-audio-mini`.
|
||||
# Selecting them from the picker would 4xx at request time.
|
||||
# `gpt-audio` and `gpt-4o-audio-preview` are kept -- they
|
||||
# stream over Chat Completions / Responses.
|
||||
r"|^gpt-realtime(?:$|-)"
|
||||
r"|^gpt-4o(?:-mini)?-realtime\b"
|
||||
r"|^gpt-audio-mini\b"
|
||||
# Legacy completion bases -- ^-anchored to avoid false
|
||||
# positives on hypothetical future chat ids containing
|
||||
# those words mid-string.
|
||||
r"|^(?:babbage|davinci|ada|curie)\b"
|
||||
r"|^text-(?:embedding|moderation|davinci|curie|babbage|ada)\b"
|
||||
# Standalone search API (separate endpoint shape).
|
||||
# `gpt-4o-search-preview` (chat-with-search) is intentionally
|
||||
# NOT matched here.
|
||||
r"|-search-api(?:-\d{4}-\d{2}-\d{2})?$"
|
||||
# Canonical non-chat prefixes.
|
||||
r"|^dall-e\b"
|
||||
r"|^computer-use\b"
|
||||
# Fine-tunes carry the user's tenant in the id.
|
||||
r"|^ft:"
|
||||
# Dated snapshot suffixes hide behind the canonical id
|
||||
# which is also in the listing. Covers both the modern
|
||||
# `-YYYY-MM-DD` dated form and the legacy compact
|
||||
# 4-digit `MMDD` form (e.g. `gpt-3.5-turbo-0125`,
|
||||
# `gpt-4-0613`, `gpt-4-1106-preview`) so the picker only
|
||||
# surfaces the canonical id even when the listing still
|
||||
# returns the snapshot copy.
|
||||
r"|-\d{4}-\d{2}-\d{2}$"
|
||||
r"|-\d{4}(?:-preview)?$"
|
||||
),
|
||||
},
|
||||
"anthropic": {
|
||||
"display_name": "Anthropic",
|
||||
|
|
@ -42,16 +117,17 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
|
|||
"claude-opus-4-7",
|
||||
"claude-opus-4-6",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-5",
|
||||
"claude-sonnet-4-5",
|
||||
"claude-haiku-4-5",
|
||||
"claude-opus-4-5-20251101",
|
||||
"claude-sonnet-4-5-20250929",
|
||||
"claude-haiku-4-5-20251001",
|
||||
],
|
||||
# Anthropic /v1/models returns dated snapshot ids alongside the
|
||||
# canonical names (e.g. claude-3-5-sonnet-20241022). Hide the
|
||||
# YYYYMMDD-suffixed variants from the picker — same intent as the
|
||||
# OpenAI denylist, just a different date format (no dashes between
|
||||
# year/month/day).
|
||||
"model_id_denylist": re.compile(r"-\d{8}$"),
|
||||
# Anthropic's /v1/models returns dated ids for every model in the
|
||||
# pre-4.6 generation -- `claude-opus-4-5-20251101`,
|
||||
# `claude-haiku-4-5-20251001`, `claude-sonnet-4-5-20250929`,
|
||||
# `claude-opus-4-1-20250805`. Per the models overview, those
|
||||
# dated ids ARE the canonical names for that generation, not
|
||||
# snapshots to hide. Dropping the previous `-\d{8}$` denylist
|
||||
# so every live model the API returns reaches the picker.
|
||||
"supports_streaming": True,
|
||||
"supports_vision": True,
|
||||
"supports_tool_calling": False,
|
||||
|
|
|
|||
|
|
@ -101,9 +101,11 @@ def test_responses_request_body_uses_input_and_instructions(monkeypatch):
|
|||
assert body["input"] == [{"role": "user", "content": "Hi"}]
|
||||
assert body["max_output_tokens"] == 512
|
||||
assert body["stream"] is True
|
||||
# Responses API on reasoning-class models (gpt-5.x / o3 / gpt-4.5 — the
|
||||
# only OpenAI ids the registry allowlist exposes) rejects these as
|
||||
# `Unsupported parameter`. Make sure we never silently forward them.
|
||||
# gpt-5.5 is reasoning-class: Responses API rejects temperature
|
||||
# and top_p with `Unsupported parameter` 400s, so the helper must
|
||||
# drop them. Non-reasoning models (gpt-4o*, gpt-4.1*, gpt-3.5-turbo*,
|
||||
# gpt-audio*, gpt-realtime*) are covered in a sibling test that
|
||||
# asserts the inverse forwarding.
|
||||
assert "temperature" not in body
|
||||
assert "top_p" not in body
|
||||
assert "presence_penalty" not in body
|
||||
|
|
@ -112,6 +114,104 @@ def test_responses_request_body_uses_input_and_instructions(monkeypatch):
|
|||
assert "messages" not in body
|
||||
|
||||
|
||||
def test_responses_forwards_sampling_for_non_reasoning_chat_families(monkeypatch):
|
||||
# Codex P1 follow-up to the picker filter change (PR 5684): the
|
||||
# OpenAI denylist now admits non-reasoning chat families
|
||||
# (gpt-4o*, gpt-4.1*, gpt-3.5-turbo*, gpt-audio*, gpt-realtime*).
|
||||
# The Responses API ACCEPTS temperature and top_p on those, so
|
||||
# the helper must forward the user's slider settings instead of
|
||||
# silently dropping them.
|
||||
for model in (
|
||||
"gpt-4o",
|
||||
"gpt-4o-mini",
|
||||
"gpt-4o-2026-01-01",
|
||||
"gpt-4.1",
|
||||
"gpt-4",
|
||||
"gpt-3.5-turbo",
|
||||
"gpt-audio",
|
||||
"gpt-realtime",
|
||||
):
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request, _captured = captured) -> httpx.Response:
|
||||
_captured["body"] = json.loads(request.content.decode("utf-8"))
|
||||
return httpx.Response(
|
||||
200,
|
||||
content = _responses_sse(
|
||||
[{"type": "response.completed", "response": {}}]
|
||||
),
|
||||
headers = {"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
_mock_http_client(monkeypatch, handler)
|
||||
|
||||
async def run():
|
||||
client = _make_client()
|
||||
async for _ in client._stream_openai_responses(
|
||||
messages = [{"role": "user", "content": "Hi"}],
|
||||
model = model,
|
||||
temperature = 0.42,
|
||||
top_p = 0.85,
|
||||
max_tokens = 32,
|
||||
enable_thinking = None,
|
||||
reasoning_effort = None,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
body = captured["body"]
|
||||
assert body["temperature"] == 0.42, (model, body)
|
||||
assert body["top_p"] == 0.85, (model, body)
|
||||
|
||||
|
||||
def test_responses_still_drops_sampling_for_reasoning_families(monkeypatch):
|
||||
# Sanity: parametrise across the reasoning-class families so a
|
||||
# future regex tweak that accidentally weakens the drop surfaces
|
||||
# here.
|
||||
for model in (
|
||||
"gpt-5.5",
|
||||
"gpt-5.4",
|
||||
"gpt-5",
|
||||
"o1",
|
||||
"o3-mini",
|
||||
"o4-mini",
|
||||
"gpt-4.5-preview",
|
||||
):
|
||||
captured: dict = {}
|
||||
|
||||
def handler(request: httpx.Request, _captured = captured) -> httpx.Response:
|
||||
_captured["body"] = json.loads(request.content.decode("utf-8"))
|
||||
return httpx.Response(
|
||||
200,
|
||||
content = _responses_sse(
|
||||
[{"type": "response.completed", "response": {}}]
|
||||
),
|
||||
headers = {"content-type": "text/event-stream"},
|
||||
)
|
||||
|
||||
_mock_http_client(monkeypatch, handler)
|
||||
|
||||
async def run():
|
||||
client = _make_client()
|
||||
async for _ in client._stream_openai_responses(
|
||||
messages = [{"role": "user", "content": "Hi"}],
|
||||
model = model,
|
||||
temperature = 0.42,
|
||||
top_p = 0.85,
|
||||
max_tokens = 32,
|
||||
enable_thinking = None,
|
||||
reasoning_effort = None,
|
||||
):
|
||||
pass
|
||||
await client.close()
|
||||
|
||||
_drive(run())
|
||||
body = captured["body"]
|
||||
assert "temperature" not in body, (model, body)
|
||||
assert "top_p" not in body, (model, body)
|
||||
|
||||
|
||||
def test_responses_translates_image_parts(monkeypatch):
|
||||
captured: dict = {}
|
||||
|
||||
|
|
|
|||
332
studio/backend/tests/test_provider_registry_filters.py
Normal file
332
studio/backend/tests/test_provider_registry_filters.py
Normal file
|
|
@ -0,0 +1,332 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
|
||||
|
||||
"""Provider registry model-id filter regression tests.
|
||||
|
||||
The OpenAI ``model_id_allowlist`` previously hardcoded the gpt-5.3/4/5
|
||||
families plus gpt-4.5 / o3 -- silently dropping every future family
|
||||
OpenAI shipped. Anthropic's ``model_id_denylist`` previously stripped
|
||||
every dated id, hiding the canonical names of every pre-4.6 model
|
||||
(Opus 4.5, Sonnet 4.5, Haiku 4.5, Opus 4.1, the 4.0 family).
|
||||
|
||||
These tests pin the new non-chat denylist (OpenAI) and the empty
|
||||
denylist (Anthropic) by walking realistic ``/v1/models`` listings
|
||||
through ``PROVIDER_REGISTRY`` and asserting the surviving set.
|
||||
"""
|
||||
|
||||
from core.inference.providers import PROVIDER_REGISTRY
|
||||
|
||||
|
||||
def _apply(provider_type: str, candidate_ids: list[str]) -> list[str]:
|
||||
"""Mirror the filter logic in ``routes/providers.list_models``."""
|
||||
info = PROVIDER_REGISTRY[provider_type]
|
||||
out = list(candidate_ids)
|
||||
allow = info.get("model_id_allowlist")
|
||||
if allow is not None:
|
||||
out = [m for m in out if allow.match(m)]
|
||||
deny = info.get("model_id_denylist")
|
||||
if deny is not None:
|
||||
out = [m for m in out if not deny.search(m)]
|
||||
return out
|
||||
|
||||
|
||||
# ── OpenAI: non-chat denylist drops only non-chat ids ──────────────
|
||||
|
||||
|
||||
def test_openai_keeps_every_known_chat_family():
|
||||
live = [
|
||||
# Current generation (must survive).
|
||||
"gpt-5.5",
|
||||
"gpt-5.5-pro",
|
||||
"gpt-5.4",
|
||||
"gpt-5.4-pro",
|
||||
"gpt-5.4-mini",
|
||||
"gpt-5.4-nano",
|
||||
"gpt-5.3-codex",
|
||||
"gpt-5.3-chat-latest",
|
||||
"o3",
|
||||
"o3-pro",
|
||||
"o3-mini",
|
||||
"o3-deep-research",
|
||||
# `gpt-4o-search-preview` and the mini variant are chat-with-
|
||||
# retrieval models that respond to standard /v1/chat/completions
|
||||
# and absolutely belong in the picker. The previous regex
|
||||
# dropped them as a false positive of `(?:^|-)search`; the
|
||||
# tightened denylist intentionally only catches the standalone
|
||||
# search-api suffix.
|
||||
"gpt-4o-search-preview",
|
||||
"gpt-4o-mini-search-preview",
|
||||
# gpt-audio is chat-completion-capable with streaming. The
|
||||
# `-mini` variant does NOT stream (Realtime API only) and is
|
||||
# asserted dropped in the realtime/audio split test below.
|
||||
"gpt-audio",
|
||||
"gpt-audio-1.5",
|
||||
# Hypothetical future families that the old allowlist would have
|
||||
# silently dropped -- they MUST surface under the new denylist.
|
||||
"gpt-5.6",
|
||||
"gpt-5.6-mini",
|
||||
"gpt-6",
|
||||
"gpt-6-pro",
|
||||
"o4",
|
||||
"o4-pro",
|
||||
"o5",
|
||||
]
|
||||
surviving = _apply("openai", live)
|
||||
assert surviving == live, surviving
|
||||
|
||||
|
||||
def test_openai_audio_and_realtime_families_split_on_streaming_support():
|
||||
# Pin the chat/non-chat split: Studio always sends `stream: true`,
|
||||
# so the picker keeps the audio chat families (gpt-audio* without
|
||||
# -mini, plus gpt-4o-audio-preview*) which stream over
|
||||
# /v1/chat/completions and /v1/responses, but drops the entire
|
||||
# Realtime family (gpt-realtime*, gpt-4o-realtime-preview*,
|
||||
# gpt-4o-mini-realtime-preview*) and gpt-audio-mini, which OpenAI's
|
||||
# model cards mark Streaming: Not supported (Realtime API only,
|
||||
# WebRTC/WebSocket transport).
|
||||
kept = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-audio",
|
||||
"gpt-audio-1.5",
|
||||
"gpt-4o-audio-preview",
|
||||
"gpt-4o-audio-preview-2024-12-17",
|
||||
],
|
||||
)
|
||||
assert kept == [
|
||||
"gpt-audio",
|
||||
"gpt-audio-1.5",
|
||||
"gpt-4o-audio-preview",
|
||||
# Dated audio-preview snapshot is dropped by the date-suffix rule;
|
||||
# the canonical id above survives.
|
||||
], kept
|
||||
|
||||
dropped = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-audio-mini",
|
||||
"gpt-realtime",
|
||||
"gpt-realtime-mini",
|
||||
"gpt-realtime-1.5",
|
||||
"gpt-realtime-2",
|
||||
"gpt-4o-realtime-preview",
|
||||
"gpt-4o-mini-realtime-preview",
|
||||
"gpt-4o-transcribe",
|
||||
"gpt-4o-mini-transcribe",
|
||||
],
|
||||
)
|
||||
assert dropped == [], dropped
|
||||
|
||||
|
||||
def test_openai_drops_non_chat_ids():
|
||||
noise = [
|
||||
# Embeddings / TTS / image / moderation / whisper / etc.
|
||||
# gpt-audio (no -mini) and gpt-4o-audio-preview are intentionally
|
||||
# OMITTED from this list -- they DO stream chat. See
|
||||
# test_openai_audio_and_realtime_families_split_on_streaming_support.
|
||||
"text-embedding-3-small",
|
||||
"text-embedding-3-large",
|
||||
"text-embedding-ada-002",
|
||||
"text-moderation-latest",
|
||||
"text-moderation-stable",
|
||||
"tts-1",
|
||||
"tts-1-hd",
|
||||
"gpt-4o-tts",
|
||||
"whisper-1",
|
||||
"dall-e-2",
|
||||
"dall-e-3",
|
||||
"gpt-image-1",
|
||||
"gpt-image-2",
|
||||
"gpt-image-1-mini",
|
||||
"chatgpt-image-latest",
|
||||
"gpt-4o-transcribe",
|
||||
"gpt-4o-mini-transcribe",
|
||||
"gpt-4o-mini-tts",
|
||||
"omni-moderation-latest",
|
||||
# Standalone search API endpoint (distinct from the
|
||||
# `*-search-preview` chat models above).
|
||||
"gpt-5-search-api",
|
||||
"gpt-5-search-api-2025-10-14",
|
||||
# Video generation.
|
||||
"sora-2",
|
||||
"sora-2-pro",
|
||||
# Computer-use is an agentic harness, not a chat id.
|
||||
"computer-use-preview",
|
||||
# Legacy bases.
|
||||
"babbage-002",
|
||||
"davinci-002",
|
||||
"text-davinci-003",
|
||||
"text-curie-001",
|
||||
"text-ada-001",
|
||||
# Fine-tunes.
|
||||
"ft:gpt-4o-mini:acme:abc:xyz",
|
||||
# Dated snapshots are still hidden.
|
||||
"gpt-4o-2024-08-06",
|
||||
"gpt-4o-mini-2024-07-18",
|
||||
"gpt-5.5-2026-04-23",
|
||||
]
|
||||
surviving = _apply("openai", noise)
|
||||
assert surviving == [], surviving
|
||||
|
||||
|
||||
def test_openai_realtime_translate_variants_are_dropped_but_parent_chat_family_survives():
|
||||
"""gpt-audio is chat-capable (streams over /v1/chat/completions)
|
||||
and stays, but the audio-translation variants share the chat
|
||||
picker's transport and would 4xx, so they must be filtered. The
|
||||
full Realtime family (gpt-realtime*, gpt-4o*-realtime-preview*)
|
||||
plus gpt-audio-mini are Realtime-only and dropped by a sibling
|
||||
assertion."""
|
||||
kept = _apply("openai", ["gpt-audio"])
|
||||
assert kept == ["gpt-audio"], kept
|
||||
|
||||
dropped = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-realtime-translate",
|
||||
"gpt-realtime-translate-mini",
|
||||
"gpt-audio-translate",
|
||||
"gpt-4o-realtime-translate-preview",
|
||||
],
|
||||
)
|
||||
assert dropped == [], dropped
|
||||
|
||||
|
||||
def test_openai_legacy_instruct_completion_ids_are_dropped():
|
||||
"""Legacy `*-instruct` completion-only ids (gpt-3.5-turbo-instruct
|
||||
and friends) speak /v1/completions, not chat/responses. Our OpenAI
|
||||
bridge only knows the chat/responses transport, so admitting them
|
||||
into the picker would 4xx every selection."""
|
||||
dropped = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-3.5-turbo-instruct",
|
||||
"gpt-3.5-turbo-instruct-0914",
|
||||
"davinci-002-instruct",
|
||||
],
|
||||
)
|
||||
assert dropped == [], dropped
|
||||
|
||||
|
||||
def test_openai_legacy_compact_snapshot_suffixes_are_dropped():
|
||||
"""Legacy `-MMDD` snapshot suffixes (gpt-3.5-turbo-0125,
|
||||
gpt-4-0613, gpt-4-1106-preview, etc.) hide behind the canonical
|
||||
id which the listing also returns; surface only the canonical so
|
||||
users do not pick a deprecated snapshot by accident. The
|
||||
`-\\d{4}(?:-preview)?$` rule must not catch canonical ids whose
|
||||
minor version happens to be a year-like number (e.g. gpt-4.5,
|
||||
o3) -- those are tested as KEEP below."""
|
||||
dropped = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-3.5-turbo-0125",
|
||||
"gpt-3.5-turbo-0301",
|
||||
"gpt-3.5-turbo-16k-0613",
|
||||
"gpt-4-0613",
|
||||
"gpt-4-0314",
|
||||
"gpt-4-32k-0613",
|
||||
"gpt-4-1106-preview",
|
||||
"gpt-4-0125-preview",
|
||||
],
|
||||
)
|
||||
assert dropped == [], dropped
|
||||
|
||||
# Canonical chat ids that share a digit-heavy tail must survive.
|
||||
kept = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-3.5-turbo",
|
||||
"gpt-4o",
|
||||
"gpt-4.5",
|
||||
"gpt-5.5",
|
||||
"gpt-5.5-mini",
|
||||
"gpt-5.5-pro",
|
||||
"o3",
|
||||
],
|
||||
)
|
||||
assert set(kept) >= {
|
||||
"gpt-3.5-turbo",
|
||||
"gpt-4o",
|
||||
"gpt-4.5",
|
||||
"gpt-5.5",
|
||||
"gpt-5.5-mini",
|
||||
"gpt-5.5-pro",
|
||||
"o3",
|
||||
}, kept
|
||||
|
||||
|
||||
def test_openai_search_preview_is_kept_search_api_is_dropped():
|
||||
# Pin the search-vs-search-api distinction so a future regex tweak
|
||||
# doesn't silently regress to dropping chat-with-search models.
|
||||
kept = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-4o-search-preview",
|
||||
"gpt-4o-mini-search-preview",
|
||||
],
|
||||
)
|
||||
assert kept == [
|
||||
"gpt-4o-search-preview",
|
||||
"gpt-4o-mini-search-preview",
|
||||
], kept
|
||||
|
||||
dropped = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-5-search-api",
|
||||
"gpt-5-search-api-2025-10-14",
|
||||
],
|
||||
)
|
||||
assert dropped == [], dropped
|
||||
|
||||
|
||||
def test_openai_legacy_completion_names_only_match_at_id_start():
|
||||
# Hypothetical future chat ids that happen to contain a legacy
|
||||
# completion-family name mid-string must NOT be dropped. The
|
||||
# `^(?:babbage|davinci|ada|curie)\b` anchor is what makes this safe.
|
||||
kept = _apply(
|
||||
"openai",
|
||||
[
|
||||
"gpt-7-davinci-edition",
|
||||
"gpt-7-ada-chat",
|
||||
"gpt-7-curie-pro",
|
||||
"gpt-7-babbage-mini",
|
||||
],
|
||||
)
|
||||
assert kept == [
|
||||
"gpt-7-davinci-edition",
|
||||
"gpt-7-ada-chat",
|
||||
"gpt-7-curie-pro",
|
||||
"gpt-7-babbage-mini",
|
||||
], kept
|
||||
# ...but the actual legacy-base ids stay dropped.
|
||||
dropped = _apply(
|
||||
"openai",
|
||||
["babbage-002", "davinci-002", "text-davinci-003"],
|
||||
)
|
||||
assert dropped == [], dropped
|
||||
|
||||
|
||||
# ── Anthropic: empty denylist; dated ids ARE canonical ───────────────
|
||||
|
||||
|
||||
def test_anthropic_surfaces_every_live_model_including_dated_ids():
|
||||
# The full set of ids /v1/models returns today.
|
||||
live = [
|
||||
"claude-opus-4-7",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-6",
|
||||
"claude-opus-4-5-20251101",
|
||||
"claude-sonnet-4-5-20250929",
|
||||
"claude-haiku-4-5-20251001",
|
||||
"claude-opus-4-1-20250805",
|
||||
"claude-opus-4-20250514",
|
||||
"claude-sonnet-4-20250514",
|
||||
]
|
||||
surviving = _apply("anthropic", live)
|
||||
assert surviving == live, surviving
|
||||
|
||||
|
||||
def test_anthropic_default_models_match_filter():
|
||||
info = PROVIDER_REGISTRY["anthropic"]
|
||||
surviving = _apply("anthropic", list(info["default_models"]))
|
||||
assert surviving == list(info["default_models"]), surviving
|
||||
Loading…
Add table
Add a link
Reference in a new issue