From abeabc71bb3bb85e0d006f00a9c665842ddb820c Mon Sep 17 00:00:00 2001 From: Lee Jackson <130007945+Imagineer99@users.noreply.github.com> Date: Wed, 20 May 2026 12:06:06 +0100 Subject: [PATCH] Studio: expand Connections model picker for local inference server (#5643) * feat: add custom model v1/model loading * fix: require base URL for local model catalog loading --------- Co-authored-by: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> --- .../core/inference/external_provider.py | 61 +++++- studio/backend/core/inference/providers.py | 30 +++ .../features/chat/chat-providers-dialog.tsx | 175 ++++++++++++------ .../src/features/chat/external-providers.ts | 34 ++++ 4 files changed, 247 insertions(+), 53 deletions(-) diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py index 16caed7858..3d5a5a6b01 100644 --- a/studio/backend/core/inference/external_provider.py +++ b/studio/backend/core/inference/external_provider.py @@ -433,6 +433,12 @@ class ExternalProviderClient: if response.status_code != 200: error_body = await response.aread() error_text = error_body.decode("utf-8", errors = "replace") + error_text = _friendly_provider_error_text( + self.provider_type, + response.status_code, + error_text, + model = model, + ) logger.error( "External provider returned %d: %s", response.status_code, @@ -2933,12 +2939,31 @@ class ExternalProviderClient: response.raise_for_status() data = response.json() # OpenAI format: {"data": [{"id": "...", ...}, ...]} - models = data.get("data", []) + # Some local servers (Ollama with no models) return data: null. + models = data.get("data") or [] + if not models and self.provider_type == "ollama": + models = await self._list_ollama_native_models() return models except httpx.HTTPError as exc: logger.error("Failed to list models from %s: %s", self.provider_type, exc) raise + async def _list_ollama_native_models(self) -> list[dict[str, Any]]: + """Fallback when Ollama's /v1/models returns an empty or null catalog.""" + root = self.base_url.removesuffix("/v1").rstrip("/") + response = await _http_client.get( + f"{root}/api/tags", + headers = self._auth_headers(), + timeout = self._timeout, + ) + response.raise_for_status() + payload = response.json() + return [ + {"id": entry.get("name", "").strip(), "owned_by": "ollama"} + for entry in (payload.get("models") or []) + if isinstance(entry, dict) and entry.get("name", "").strip() + ] + async def verify_models_endpoint_lightweight(self) -> None: """ Confirm GET /models returns 200 without buffering the full response body. @@ -3087,6 +3112,40 @@ class ExternalProviderClient: """No-op — the underlying client is shared across requests.""" +def _provider_display_name(provider_type: str) -> str: + from core.inference.providers import get_provider_info + + info = get_provider_info(provider_type) or {} + return str(info.get("display_name") or provider_type) + + +def _friendly_provider_error_text( + provider_type: str, + status_code: int, + raw_message: str, + *, + model: str | None = None, +) -> str: + """Rewrite common provider errors into actionable Studio copy.""" + if status_code == 404 and model: + lowered = raw_message.lower() + if "not found" in lowered or "not_found" in lowered: + if provider_type == "ollama": + label = _provider_display_name(provider_type) + return ( + f"Model '{model}' is not installed in {label}. " + f"Run `ollama pull {model}` in a terminal, then retry." + ) + if provider_type in ("vllm", "llama_cpp"): + label = _provider_display_name(provider_type) + return ( + f"Model '{model}' is not available on the {label} server. " + "Check that the server is running and the model is loaded, " + "then retry." + ) + return raw_message + + def _error_sse_line(status_code: int, message: str, provider_type: str) -> str: """Format an error as an SSE data line in OpenAI error format.""" import json diff --git a/studio/backend/core/inference/providers.py b/studio/backend/core/inference/providers.py index 143ced95f1..fef9ba3e12 100644 --- a/studio/backend/core/inference/providers.py +++ b/studio/backend/core/inference/providers.py @@ -240,6 +240,36 @@ PROVIDER_REGISTRY: dict[str, dict[str, Any]] = { # /api/providers/registry dropdown — see list_available_providers. "hidden": True, }, + "ollama": { + "display_name": "Ollama", + "base_url": "http://localhost:11434/v1", + "default_models": [], + "supports_streaming": True, + "supports_vision": True, + "supports_tool_calling": True, + "auth_header": "Authorization", + "auth_prefix": "Bearer ", + "notes": ( + "Local Ollama server. OpenAI-compatible /v1/chat/completions; " + "no API key. Surfaced via CUSTOM_PROVIDER_PRESETS in the frontend." + ), + "hidden": True, + }, + "llama_cpp": { + "display_name": "llama.cpp", + "base_url": "http://localhost:8080/v1", + "default_models": [], + "supports_streaming": True, + "supports_vision": True, + "supports_tool_calling": True, + "auth_header": "Authorization", + "auth_prefix": "Bearer ", + "notes": ( + "Local llama.cpp server (llama-server). OpenAI-compatible " + "/v1/chat/completions. Surfaced via CUSTOM_PROVIDER_PRESETS." + ), + "hidden": True, + }, "openrouter": { "display_name": "OpenRouter", "base_url": "https://openrouter.ai/api/v1", diff --git a/studio/frontend/src/features/chat/chat-providers-dialog.tsx b/studio/frontend/src/features/chat/chat-providers-dialog.tsx index dcd51da044..dca06b03e5 100644 --- a/studio/frontend/src/features/chat/chat-providers-dialog.tsx +++ b/studio/frontend/src/features/chat/chat-providers-dialog.tsx @@ -51,9 +51,11 @@ import type { ExternalProviderConfig } from "./external-providers"; import { CUSTOM_BACKEND_PROVIDER_TYPE, CUSTOM_PROVIDER_PRESETS, + allowsManualModelIdsWithCatalog, customProviderBaseUrlPlaceholder, customProviderDisplayName, customProviderModelIdsPlaceholder, + customPresetSkipsApiKeyField, getExternalProviderApiKey, isCustomProviderType, LEGACY_CUSTOM_PROVIDER_TYPE, @@ -61,6 +63,7 @@ import { setExternalProviderApiKey, supportsProviderPromptCaching, supportsProviderReasoningToggle, + supportsRemoteModelCatalog, toExternalBackendProviderType, } from "./external-providers"; import { useExternalProvidersStore } from "./stores/external-providers-store"; @@ -137,8 +140,37 @@ function parseManualModelIds(text: string): string[] { return out; } -// Remote providers safe for manual model IDs (openrouter drops unused params). -const MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES = new Set(["openrouter"]); +// Remote providers that support both catalog load and manual model IDs. +const EMPTY_CATALOG_HINTS: Record = + { + ollama: { + title: "No local Ollama models found.", + description: + "Run `ollama pull ` in a terminal, then reload — or enter a model ID manually below.", + }, + llama_cpp: { + title: "No llama.cpp models found.", + description: + "Ensure llama-server is running with models loaded, then reload — or enter model IDs manually below.", + }, + vllm: { + title: "No vLLM models found.", + description: + "Ensure the vLLM server is running and models are loaded, then reload — or enter model IDs manually below.", + }, + }; + +function emptyCatalogHint(providerType: string): { + title: string; + description: string; +} { + return ( + EMPTY_CATALOG_HINTS[providerType] ?? { + title: "No models returned by this provider.", + description: "Enter model IDs manually below, or check the server.", + } + ); +} function pruneProviderModelIds(providerType: string, modelIds: string[]): string[] { if (providerType === "anthropic") { @@ -199,11 +231,9 @@ export function ChatProvidersSettings({ (s) => s.setConnectionsEnabled, ); const isCustomProvider = isCustomProviderType(providerType); - // Ollama runs locally and does not require an API key. Hide the input - // entirely rather than just marking it optional so users aren't prompted - // for a credential the provider never uses. - const isOllamaProvider = providerType === "ollama"; - const showApiKeyField = !isOllamaProvider; + // Local presets (Ollama, llama.cpp) never use API keys — hide the field. + // vLLM may optionally use a bearer token on secured deployments. + const showApiKeyField = !customPresetSkipsApiKeyField(providerType); const showReasoningToggle = supportsProviderReasoningToggle(providerType); const registryByType = useMemo( @@ -213,22 +243,31 @@ export function ChatProvidersSettings({ const isCuratedModelList = useMemo(() => { return registryByType.get(providerType)?.model_list_mode === "curated"; }, [registryByType, providerType]); - const isManualModelList = isCustomProvider || isCuratedModelList; + const isManualModelList = + (isCustomProvider && !supportsRemoteModelCatalog(providerType)) || + isCuratedModelList; const modelsPanelKey = isCustomProvider ? providerType || "custom" : isCuratedModelList ? "curated" : "remote"; - const formModelCount = isManualModelList - ? new Set([...selectedModelIds, ...parseManualModelIds(manualModelIds)]) - .size - : selectedModelIds.length; + const remoteAllowsManual = allowsManualModelIdsWithCatalog(providerType); + const formModelCount = + isManualModelList || remoteAllowsManual + ? new Set([...selectedModelIds, ...parseManualModelIds(manualModelIds)]) + .size + : selectedModelIds.length; const modelStatusLabel = - !isManualModelList && availableModels.length === 0 + !isManualModelList && + !remoteAllowsManual && + availableModels.length === 0 ? "No models loaded" : `${formModelCount} ${formModelCount === 1 ? "model" : "models"} selected`; - const showModelsBody = isManualModelList || availableModels.length > 0; + const showModelsBody = + isManualModelList || + remoteAllowsManual || + availableModels.length > 0; const filteredAvailableModels = useMemo(() => { const query = modelSearchQuery.trim().toLowerCase(); if (!query) { @@ -257,15 +296,11 @@ export function ChatProvidersSettings({ } return; } - // Seed default_models only when the catalog is not fetched live: - // curated providers (catalog too large to enumerate, defaults are - // the suggestion shortlist) and Ollama (local, no API key — local - // /models stands in). Remote-mode cloud providers stay empty until - // the user clicks "Load available models" with a key, since - // different API tiers expose different catalogs and we don't want - // to advertise models the user can't actually call. - const seedDefaults = - entry.model_list_mode === "curated" || providerType === "ollama"; + // Seed default_models only for curated providers (catalog too large to + // enumerate — defaults are the suggestion shortlist). Remote-mode cloud + // providers and local OpenAI-compat presets stay empty until the user + // clicks "Load available models". + const seedDefaults = entry.model_list_mode === "curated"; setAvailableModels(seedDefaults ? [...entry.default_models] : []); setSelectedModelIds([]); setManualModelIds(""); @@ -379,12 +414,8 @@ export function ChatProvidersSettings({ function openAddProvider() { resetForm(); const entry = providerType ? registryByType.get(providerType) : null; - if (entry) { - const seedDefaults = - entry.model_list_mode === "curated" || providerType === "ollama"; - if (seedDefaults) { - setAvailableModels([...entry.default_models]); - } + if (entry?.model_list_mode === "curated") { + setAvailableModels([...entry.default_models]); } setPage("form"); } @@ -444,7 +475,7 @@ export function ChatProvidersSettings({ toast.error("Choose a provider first."); return; } - if (isCustomProvider) { + if (isCustomProvider && !supportsRemoteModelCatalog(providerType)) { toast.info("This connection uses manual model IDs."); return; } @@ -460,14 +491,20 @@ export function ChatProvidersSettings({ } setModelsLoading(true); try { - const baseUrl = parseBaseUrlForProvider(baseUrlDraft, isCustomProvider); + const baseUrl = parseBaseUrlForProvider( + baseUrlDraft, + supportsRemoteModelCatalog(providerType), + ); + const backendProviderType = + toExternalBackendProviderType(providerType) ?? providerType; const models = await listProviderModels({ - providerType, + providerType: backendProviderType, apiKey: apiKey.trim(), baseUrl, }); - const registryDefaults = - registryByType.get(providerType)?.default_models ?? []; + const registryDefaults = supportsRemoteModelCatalog(providerType) + ? [] + : (registryByType.get(providerType)?.default_models ?? []); // Union of registry defaults + fetched models, defaults first so any // curated picks (e.g. claude-haiku-4-5) always show even when the // provider's /models endpoint omits them. @@ -483,6 +520,14 @@ export function ChatProvidersSettings({ setSelectedModelIds((prev) => prev.filter((id) => modelIds.includes(id)), ); + if (modelIds.length === 0) { + const hint = emptyCatalogHint(providerType); + toast.info(hint.title, { description: hint.description }); + } else { + toast.success( + `Found ${modelIds.length} ${modelIds.length === 1 ? "model" : "models"}.`, + ); + } if (editingProviderId) { onProvidersChange( providersRef.current.map((provider) => @@ -516,9 +561,10 @@ export function ChatProvidersSettings({ return; } const curated = selectedRegistryEntry?.model_list_mode === "curated"; - const manualOnly = isCustomProvider || curated; - const remoteAllowsManual = - MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has(providerType); + const manualOnly = + (isCustomProvider && !supportsRemoteModelCatalog(providerType)) || + curated; + const remoteAllowsManual = allowsManualModelIdsWithCatalog(providerType); const manualIds = parseManualModelIds(manualModelIds); const allowManual = manualOnly || remoteAllowsManual; const modelsToSave = pruneProviderModelIds( @@ -537,7 +583,7 @@ export function ChatProvidersSettings({ toast.error("Add at least one model ID."); return; } - } else if (remoteAllowsManual && manualIds.length > 0) { + } else if (remoteAllowsManual) { if (modelsToSave.length === 0) { toast.error("Add at least one model ID."); return; @@ -621,8 +667,11 @@ export function ChatProvidersSettings({ } const entry = registryByType.get(existing.providerType); const curated = entry?.model_list_mode === "curated"; - const manualOnly = isEditingCustomProvider || curated; - const remoteAllowsManual = MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has( + const manualOnly = + (isEditingCustomProvider && + !supportsRemoteModelCatalog(existing.providerType)) || + curated; + const remoteAllowsManual = allowsManualModelIdsWithCatalog( existing.providerType, ); const manualIds = parseManualModelIds(manualModelIds); @@ -643,7 +692,7 @@ export function ChatProvidersSettings({ toast.error("Add at least one model ID."); return; } - } else if (remoteAllowsManual && manualIds.length > 0) { + } else if (remoteAllowsManual) { if (modelsToSave.length === 0) { toast.error("Add at least one model ID."); return; @@ -729,12 +778,34 @@ export function ChatProvidersSettings({ ? provider.isReasoningModel === true : false, ); - if (isCustomProviderType(provider.providerType)) { + if ( + isCustomProviderType(provider.providerType) && + !supportsRemoteModelCatalog(provider.providerType) + ) { setAvailableModels([]); setSelectedModelIds([]); setManualModelIds(provider.models.join("\n")); return; } + if (supportsRemoteModelCatalog(provider.providerType)) { + const cachedCatalog = provider.availableModels ?? []; + const catalogModels = pruneProviderModelIds(provider.providerType, [ + ...new Set( + cachedCatalog + .map((model) => model.trim()) + .filter((model) => model.length > 0), + ), + ]); + setAvailableModels(catalogModels); + const catalogSet = new Set(catalogModels); + setSelectedModelIds( + provider.models.filter((model) => catalogSet.has(model)), + ); + setManualModelIds( + provider.models.filter((model) => !catalogSet.has(model)).join("\n"), + ); + return; + } const entry = registryByType.get(provider.providerType); if (entry?.model_list_mode === "curated") { const defaults = new Set(entry.default_models); @@ -779,10 +850,8 @@ export function ChatProvidersSettings({ async function testProvider(provider: ExternalProviderConfig) { const savedKey = getExternalProviderApiKey(provider.id).trim(); - // Ollama runs locally and never requires a key — fall through to the - // real connection check instead of prompting for credentials the form - // no longer exposes. - if (!savedKey && provider.providerType !== "ollama") { + // Local OpenAI-compat presets skip API keys — run the connection check. + if (!savedKey && !supportsRemoteModelCatalog(provider.providerType)) { if (isCustomProviderType(provider.providerType)) { await editProvider(provider); toast.info(CUSTOM_PROVIDER_MISSING_KEY_MESSAGE); @@ -1083,7 +1152,7 @@ export function ChatProvidersSettings({ modelsLoading || mutatingProvider || isManualModelList } title={ - isCustomProvider + isManualModelList && isCustomProvider ? "This connection uses manual model IDs" : isCuratedModelList ? "Full catalog is not fetched for this provider" @@ -1103,7 +1172,7 @@ export function ChatProvidersSettings({ )} - {isCustomProvider ? ( + {isCustomProvider && !supportsRemoteModelCatalog(providerType) ? (
) : availableModels.length === 0 && - !MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has(providerType) ? null : ( + !allowsManualModelIdsWithCatalog(providerType) ? null : (
{availableModels.length === 0 ? null : ( <> @@ -1288,8 +1357,8 @@ export function ChatProvidersSettings({ )} - {/* Manual IDs allowed for openrouter only. */} - {MANUAL_MODEL_ID_REMOTE_PROVIDER_TYPES.has(providerType) ? ( + {/* Manual IDs allowed alongside catalog load. */} + {allowsManualModelIdsWithCatalog(providerType) ? (