ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Studio: expand Connections model picker for local inference server (#5643)

Browse source 
* feat: add custom model v1/model loading

* fix: require base URL for local model catalog loading

---------

Co-authored-by: Roland Tannous <115670425+rolandtannous@users.noreply.github.com>
This commit is contained in:
Lee Jackson 2026-05-20 12:06:06 +01:00 • committed by GitHub
commit abeabc71bb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 247 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

61
studio/backend/core/inference/external_provider.py
View file

@ -433,6 +433,12 @@ class ExternalProviderClient:
if response.status_code != 200:
error_body = await response.aread()
error_text = error_body.decode("utf-8", errors = "replace")
error_text = _friendly_provider_error_text(
self.provider_type,
response.status_code,
error_text,
model = model,
)
logger.error(
"External provider returned %d: %s",
response.status_code,
@ -2933,12 +2939,31 @@ class ExternalProviderClient:
response.raise_for_status()
data = response.json()
# OpenAI format: {"data": [{"id": "...", ...}, ...]}
models = data.get("data", [])
# Some local servers (Ollama with no models) return data: null.
models = data.get("data") or []
if not models and self.provider_type == "ollama":
models = await self._list_ollama_native_models()
return models
except httpx.HTTPError as exc:
logger.error("Failed to list models from %s: %s", self.provider_type, exc)
raise
async def _list_ollama_native_models(self) -> list[dict[str, Any]]:
"""Fallback when Ollama's /v1/models returns an empty or null catalog."""
root = self.base_url.removesuffix("/v1").rstrip("/")
response = await _http_client.get(
f"{root}/api/tags",
headers = self._auth_headers(),
timeout = self._timeout,
)
response.raise_for_status()
payload = response.json()
return [
{"id": entry.get("name", "").strip(), "owned_by": "ollama"}
for entry in (payload.get("models") or [])
if isinstance(entry, dict) and entry.get("name", "").strip()
]
async def verify_models_endpoint_lightweight(self) -> None:
"""
Confirm GET /models returns 200 without buffering the full response body.
@ -3087,6 +3112,40 @@ class ExternalProviderClient:
"""No-op — the underlying client is shared across requests."""
def _provider_display_name(provider_type: str) -> str:
from core.inference.providers import get_provider_info
info = get_provider_info(provider_type) or {}
return str(info.get("display_name") or provider_type)
def _friendly_provider_error_text(
provider_type: str,
status_code: int,
raw_message: str,
*,
model: str | None = None,
) -> str:
"""Rewrite common provider errors into actionable Studio copy."""
if status_code == 404 and model:
lowered = raw_message.lower()
if "not found" in lowered or "not_found" in lowered:
if provider_type == "ollama":
label = _provider_display_name(provider_type)
return (
f"Model '{model}' is not installed in {label}. "
f"Run `ollama pull {model}` in a terminal, then retry."
)
if provider_type in ("vllm", "llama_cpp"):
label = _provider_display_name(provider_type)
return (
f"Model '{model}' is not available on the {label} server. "
"Check that the server is running and the model is loaded, "
"then retry."
)
return raw_message
def _error_sse_line(status_code: int, message: str, provider_type: str) -> str:
"""Format an error as an SSE data line in OpenAI error format."""
import json

30
studio/backend/core/inference/providers.py
View file

@ -240,6 +240,36 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = {
# /api/providers/registry dropdown — see list_available_providers.
"hidden": True,
},
"ollama": {
"display_name": "Ollama",
"base_url": "http://localhost:11434/v1",
"default_models": [],
"supports_streaming": True,
"supports_vision": True,
"supports_tool_calling": True,
"auth_header": "Authorization",
"auth_prefix": "Bearer ",
"notes": (
"Local Ollama server. OpenAI-compatible /v1/chat/completions; "
"no API key. Surfaced via CUSTOM_PROVIDER_PRESETS in the frontend."
),
"hidden": True,
},
"llama_cpp": {
"display_name": "llama.cpp",
"base_url": "http://localhost:8080/v1",
"default_models": [],
"supports_streaming": True,
"supports_vision": True,
"supports_tool_calling": True,
"auth_header": "Authorization",
"auth_prefix": "Bearer ",
"notes": (
"Local llama.cpp server (llama-server). OpenAI-compatible "
"/v1/chat/completions. Surfaced via CUSTOM_PROVIDER_PRESETS."
),
"hidden": True,
},
"openrouter": {
"display_name": "OpenRouter",
"base_url": "https://openrouter.ai/api/v1",

175
studio/frontend/src/features/chat/chat-providers-dialog.tsx
View file

@ -51,9 +51,11 @@ import type { ExternalProviderConfig } from "./external-providers";
import {
CUSTOM_BACKEND_PROVIDER_TYPE,
CUSTOM_PROVIDER_PRESETS,
allowsManualModelIdsWithCatalog,
customProviderBaseUrlPlaceholder,
customProviderDisplayName,
customProviderModelIdsPlaceholder,
customPresetSkipsApiKeyField,
getExternalProviderApiKey,
isCustomProviderType,
LEGACY_CUSTOM_PROVIDER_TYPE,
@ -61,6 +63,7 @@ import {
setExternalProviderApiKey,
supportsProviderPromptCaching,
supportsProviderReasoningToggle,
supportsRemoteModelCatalog,
toExternalBackendProviderType,
} from "./external-providers";
import { useExternalProvidersStore } from "./stores/external-providers-store";
@ -137,8 +140,37 @@ function parseManualModelIds(text: string): string[] {
return out;
}
// Remote providers safe for manual model IDs (openrouter drops unused params).
const MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES = new Set<string>(["openrouter"]);
// Remote providers that support both catalog load and manual model IDs.
const EMPTY_CATALOG_HINTS: Record<string, { title: string; description: string }> =
{
ollama: {
title: "No local Ollama models found.",
description:
"Run `ollama pull <model>` in a terminal, then reload — or enter a model ID manually below.",
},
llama_cpp: {
title: "No llama.cpp models found.",
description:
"Ensure llama-server is running with models loaded, then reload — or enter model IDs manually below.",
},
vllm: {
title: "No vLLM models found.",
description:
"Ensure the vLLM server is running and models are loaded, then reload — or enter model IDs manually below.",
},
};
function emptyCatalogHint(providerType: string): {
title: string;
description: string;
} {
return (
EMPTY_CATALOG_HINTS[providerType] ?? {
title: "No models returned by this provider.",
description: "Enter model IDs manually below, or check the server.",
}
);
}
function pruneProviderModelIds(providerType: string, modelIds: string[]): string[] {
if (providerType === "anthropic") {
@ -199,11 +231,9 @@ export function ChatProvidersSettings({
(s) => s.setConnectionsEnabled,
);
const isCustomProvider = isCustomProviderType(providerType);
// Ollama runs locally and does not require an API key. Hide the input
// entirely rather than just marking it optional so users aren't prompted
// for a credential the provider never uses.
const isOllamaProvider = providerType === "ollama";
const showApiKeyField = !isOllamaProvider;
// Local presets (Ollama, llama.cpp) never use API keys — hide the field.
// vLLM may optionally use a bearer token on secured deployments.
const showApiKeyField = !customPresetSkipsApiKeyField(providerType);
const showReasoningToggle = supportsProviderReasoningToggle(providerType);
const registryByType = useMemo(
@ -213,22 +243,31 @@ export function ChatProvidersSettings({
const isCuratedModelList = useMemo(() => {
return registryByType.get(providerType)?.model_list_mode === "curated";
}, [registryByType, providerType]);
const isManualModelList = isCustomProvider || isCuratedModelList;
const isManualModelList =
(isCustomProvider && !supportsRemoteModelCatalog(providerType)) ||
isCuratedModelList;
const modelsPanelKey = isCustomProvider
? providerType || "custom"
: isCuratedModelList
? "curated"
: "remote";
const formModelCount = isManualModelList
? new Set([...selectedModelIds, ...parseManualModelIds(manualModelIds)])
.size
: selectedModelIds.length;
const remoteAllowsManual = allowsManualModelIdsWithCatalog(providerType);
const formModelCount =
isManualModelList || remoteAllowsManual
? new Set([...selectedModelIds, ...parseManualModelIds(manualModelIds)])
.size
: selectedModelIds.length;
const modelStatusLabel =
!isManualModelList && availableModels.length === 0
!isManualModelList &&
!remoteAllowsManual &&
availableModels.length === 0
? "No models loaded"
: `${formModelCount} ${formModelCount === 1 ? "model" : "models"} selected`;
const showModelsBody = isManualModelList || availableModels.length > 0;
const showModelsBody =
isManualModelList ||
remoteAllowsManual ||
availableModels.length > 0;
const filteredAvailableModels = useMemo(() => {
const query = modelSearchQuery.trim().toLowerCase();
if (!query) {
@ -257,15 +296,11 @@ export function ChatProvidersSettings({
}
return;
}
// Seed default_models only when the catalog is not fetched live:
// curated providers (catalog too large to enumerate, defaults are
// the suggestion shortlist) and Ollama (local, no API key — local
// /models stands in). Remote-mode cloud providers stay empty until
// the user clicks "Load available models" with a key, since
// different API tiers expose different catalogs and we don't want
// to advertise models the user can't actually call.
const seedDefaults =
entry.model_list_mode === "curated" || providerType === "ollama";
// Seed default_models only for curated providers (catalog too large to
// enumerate — defaults are the suggestion shortlist). Remote-mode cloud
// providers and local OpenAI-compat presets stay empty until the user
// clicks "Load available models".
const seedDefaults = entry.model_list_mode === "curated";
setAvailableModels(seedDefaults ? [...entry.default_models] : []);
setSelectedModelIds([]);
setManualModelIds("");
@ -379,12 +414,8 @@ export function ChatProvidersSettings({
function openAddProvider() {
resetForm();
const entry = providerType ? registryByType.get(providerType) : null;
if (entry) {
const seedDefaults =
entry.model_list_mode === "curated" || providerType === "ollama";
if (seedDefaults) {
setAvailableModels([...entry.default_models]);
}
if (entry?.model_list_mode === "curated") {
setAvailableModels([...entry.default_models]);
}
setPage("form");
}
@ -444,7 +475,7 @@ export function ChatProvidersSettings({
toast.error("Choose a provider first.");
return;
}
if (isCustomProvider) {
if (isCustomProvider && !supportsRemoteModelCatalog(providerType)) {
toast.info("This connection uses manual model IDs.");
return;
}
@ -460,14 +491,20 @@ export function ChatProvidersSettings({
}
setModelsLoading(true);
try {
const baseUrl = parseBaseUrlForProvider(baseUrlDraft, isCustomProvider);
const baseUrl = parseBaseUrlForProvider(
baseUrlDraft,
supportsRemoteModelCatalog(providerType),
);
const backendProviderType =
toExternalBackendProviderType(providerType) ?? providerType;
const models = await listProviderModels({
providerType,
providerType: backendProviderType,
apiKey: apiKey.trim(),
baseUrl,
});
const registryDefaults =
registryByType.get(providerType)?.default_models ?? [];
const registryDefaults = supportsRemoteModelCatalog(providerType)
? []
: (registryByType.get(providerType)?.default_models ?? []);
// Union of registry defaults + fetched models, defaults first so any
// curated picks (e.g. claude-haiku-4-5) always show even when the
// provider's /models endpoint omits them.
@ -483,6 +520,14 @@ export function ChatProvidersSettings({
setSelectedModelIds((prev) =>
prev.filter((id) => modelIds.includes(id)),
);
if (modelIds.length === 0) {
const hint = emptyCatalogHint(providerType);
toast.info(hint.title, { description: hint.description });
} else {
toast.success(
`Found ${modelIds.length} ${modelIds.length === 1 ? "model" : "models"}.`,
);
}
if (editingProviderId) {
onProvidersChange(
providersRef.current.map((provider) =>
@ -516,9 +561,10 @@ export function ChatProvidersSettings({
return;
}
const curated = selectedRegistryEntry?.model_list_mode === "curated";
const manualOnly = isCustomProvider || curated;
const remoteAllowsManual =
MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has(providerType);
const manualOnly =
(isCustomProvider && !supportsRemoteModelCatalog(providerType)) ||
curated;
const remoteAllowsManual = allowsManualModelIdsWithCatalog(providerType);
const manualIds = parseManualModelIds(manualModelIds);
const allowManual = manualOnly || remoteAllowsManual;
const modelsToSave = pruneProviderModelIds(
@ -537,7 +583,7 @@ export function ChatProvidersSettings({
toast.error("Add at least one model ID.");
return;
}
} else if (remoteAllowsManual && manualIds.length > 0) {
} else if (remoteAllowsManual) {
if (modelsToSave.length === 0) {
toast.error("Add at least one model ID.");
return;
@ -621,8 +667,11 @@ export function ChatProvidersSettings({
}
const entry = registryByType.get(existing.providerType);
const curated = entry?.model_list_mode === "curated";
const manualOnly = isEditingCustomProvider || curated;
const remoteAllowsManual = MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has(
const manualOnly =
(isEditingCustomProvider &&
!supportsRemoteModelCatalog(existing.providerType)) ||
curated;
const remoteAllowsManual = allowsManualModelIdsWithCatalog(
existing.providerType,
);
const manualIds = parseManualModelIds(manualModelIds);
@ -643,7 +692,7 @@ export function ChatProvidersSettings({
toast.error("Add at least one model ID.");
return;
}
} else if (remoteAllowsManual && manualIds.length > 0) {
} else if (remoteAllowsManual) {
if (modelsToSave.length === 0) {
toast.error("Add at least one model ID.");
return;
@ -729,12 +778,34 @@ export function ChatProvidersSettings({
? provider.isReasoningModel === true
: false,
);
if (isCustomProviderType(provider.providerType)) {
if (
isCustomProviderType(provider.providerType) &&
!supportsRemoteModelCatalog(provider.providerType)
) {
setAvailableModels([]);
setSelectedModelIds([]);
setManualModelIds(provider.models.join("\n"));
return;
}
if (supportsRemoteModelCatalog(provider.providerType)) {
const cachedCatalog = provider.availableModels ?? [];
const catalogModels = pruneProviderModelIds(provider.providerType, [
...new Set(
cachedCatalog
.map((model) => model.trim())
.filter((model) => model.length > 0),
),
]);
setAvailableModels(catalogModels);
const catalogSet = new Set(catalogModels);
setSelectedModelIds(
provider.models.filter((model) => catalogSet.has(model)),
);
setManualModelIds(
provider.models.filter((model) => !catalogSet.has(model)).join("\n"),
);
return;
}
const entry = registryByType.get(provider.providerType);
if (entry?.model_list_mode === "curated") {
const defaults = new Set(entry.default_models);
@ -779,10 +850,8 @@ export function ChatProvidersSettings({
async function testProvider(provider: ExternalProviderConfig) {
const savedKey = getExternalProviderApiKey(provider.id).trim();
// Ollama runs locally and never requires a key — fall through to the
// real connection check instead of prompting for credentials the form
// no longer exposes.
if (!savedKey && provider.providerType !== "ollama") {
// Local OpenAI-compat presets skip API keys — run the connection check.
if (!savedKey && !supportsRemoteModelCatalog(provider.providerType)) {
if (isCustomProviderType(provider.providerType)) {
await editProvider(provider);
toast.info(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE);
@ -1083,7 +1152,7 @@ export function ChatProvidersSettings({
modelsLoading || mutatingProvider || isManualModelList
}
title={
isCustomProvider
isManualModelList && isCustomProvider
? "This connection uses manual model IDs"
: isCuratedModelList
? "Full catalog is not fetched for this provider"
@ -1103,7 +1172,7 @@ export function ChatProvidersSettings({
)}
</Button>
</div>
{isCustomProvider ? (
{isCustomProvider && !supportsRemoteModelCatalog(providerType) ? (
<div className="space-y-3 px-4 py-4">
<div className="space-y-2">
<Label
@ -1219,7 +1288,7 @@ export function ChatProvidersSettings({
</div>
</div>
) : availableModels.length === 0 &&
!MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has(providerType) ? null : (
!allowsManualModelIdsWithCatalog(providerType) ? null : (
<div className="space-y-3 px-4 py-4">
{availableModels.length === 0 ? null : (
<>
@ -1288,8 +1357,8 @@ export function ChatProvidersSettings({
</ul>
</>
)}
{/* Manual IDs allowed for openrouter only. */}
{MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has(providerType) ? (
{/* Manual IDs allowed alongside catalog load. */}
{allowsManualModelIdsWithCatalog(providerType) ? (
<div className="space-y-2">
<Label
htmlFor="provider-manual-models"
@ -1305,7 +1374,9 @@ export function ChatProvidersSettings({
onChange={(event) =>
setManualModelIds(event.target.value)
}
placeholder={"model-id-1\nmodel-id-2"}
placeholder={customProviderModelIdsPlaceholder(
providerType,
)}
rows={4}
className="min-h-[80px] resize-y font-mono text-sm"
/>

34
studio/frontend/src/features/chat/external-providers.ts
View file

@ -134,6 +134,38 @@ export function isCustomProviderType(
return providerType in CUSTOM_PROVIDER_LABELS;
}
/** Local OpenAI-compat presets that expose GET /v1/models (no API key). */
const REMOTE_MODEL_CATALOG_CUSTOM_PROVIDER_TYPES = new Set([
"ollama",
"vllm",
"llama_cpp",
]);
export function supportsRemoteModelCatalog(
providerType: string | null | undefined,
): boolean {
return (
providerType != null &&
REMOTE_MODEL_CATALOG_CUSTOM_PROVIDER_TYPES.has(providerType)
);
}
/** Presets that skip the API-key field (local servers with no auth by default). */
export function customPresetSkipsApiKeyField(
providerType: string | null | undefined,
): boolean {
return providerType === "ollama" || providerType === "llama_cpp";
}
/** Catalog load plus optional manual model IDs (OpenRouter + local presets). */
export function allowsManualModelIdsWithCatalog(
providerType: string | null | undefined,
): boolean {
if (!providerType) return false;
if (providerType === "openrouter") return true;
return supportsRemoteModelCatalog(providerType);
}
export function customProviderDisplayName(
providerType: string | null | undefined,
): string {
@ -181,6 +213,8 @@ export function toExternalBackendProviderType(
// type through so the backend routes vLLM to /v1/chat/completions instead
// of the OpenAI Responses path used for gpt-5.x.
if (providerType === "vllm") return "vllm";
if (providerType === "ollama") return "ollama";
if (providerType === "llama_cpp") return "llama_cpp";
return isCustomProviderType(providerType)
? CUSTOM_BACKEND_PROVIDER_TYPE
: providerType;