From f7e8a85d32df76681a886a823e66eb303391d7ab Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Tue, 19 May 2026 05:48:59 -0700
Subject: [PATCH] studio/frontend: cap auto-load cascade attempts (#5578)

* studio/frontend: cap auto-load cascade attempts

autoLoadSmallestModel walks every cached GGUF and safetensors repo with a
try/catch + continue, so a folder of broken caches (missing files, stale
llama.cpp prebuilt, GPU OOM) can fire dozens of failing POST /api/inference/load
calls in a row. Each call costs ~5 seconds (HF metadata probe + DNS guard
inside inference.py), so the user sees a runaway sequence of request_completed
log lines after sending one message that needed an auto-load.

Cap the total loadModel calls inside autoLoadSmallestModel at 3 (GGUF cascade
plus safetensors fallback share the same counter). Caching that fails three
times in a row is almost certainly an environment problem, not "we haven't
found the working one yet"; the default-Gemma download path still runs.

No behavior change on the happy path: success returns after the first hit
exactly like today, and the trust-remote-code skip path does not consume an
attempt slot.

* shorter comment on auto-load cap

* studio chat: extend autoload cap to default Gemma fallback

Cached cascade respected MAX_AUTO_LOAD_ATTEMPTS but the default-Gemma
download path skipped the budget, so a broken cache could still emit a
fourth /api/inference/load. Gate the fallback on the same cap (and bump
loadAttempts when we do call loadModel) so the total cross-path budget
is 3, matching the cap's intent.
---
 .../src/features/chat/api/chat-adapter.ts     | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 61d71b641a..f87229d78f 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -437,6 +437,9 @@ function waitForModelReady(abortSignal?: AbortSignal): Promise<void> {
  * without selecting one. Prefers GGUF (picks smallest cached variant),
  * falls back to smallest cached safetensors model.
  */
+// Cap cascade so broken cached repos can't spam /api/inference/load.
+const MAX_AUTO_LOAD_ATTEMPTS = 3;
+
 async function autoLoadSmallestModel(): Promise<{
   loaded: boolean;
   blockedByTrustRemoteCode: boolean;
@@ -451,6 +454,7 @@ async function autoLoadSmallestModel(): Promise<{
   });
   let blockedByTrustRemoteCode = false;
   let hadNonTrustFailure = false;
+  let loadAttempts = 0;
 
   async function canAutoLoad(payload: {
     model_path: string;
@@ -481,6 +485,7 @@ async function autoLoadSmallestModel(): Promise<{
     if (ggufRepos.length > 0) {
       const sorted = [...ggufRepos].sort((a, b) => a.size_bytes - b.size_bytes);
       for (const repo of sorted) {
+        if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) break;
         try {
           const variants = await listGgufVariants(repo.repo_id);
           const downloaded = variants.variants
@@ -498,6 +503,7 @@ async function autoLoadSmallestModel(): Promise<{
             ) {
               continue;
             }
+            loadAttempts += 1;
             const loadResp = await loadModel({
               model_path: repo.repo_id,
               hf_token: hfToken,
@@ -560,6 +566,7 @@ async function autoLoadSmallestModel(): Promise<{
     if (modelRepos.length > 0) {
       const sorted = [...modelRepos].sort((a, b) => a.size_bytes - b.size_bytes);
       for (const repo of sorted) {
+        if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) break;
         try {
           if (
             !(await canAutoLoad({
@@ -571,6 +578,7 @@ async function autoLoadSmallestModel(): Promise<{
           ) {
             continue;
           }
+          loadAttempts += 1;
           const sfLoadResp = await loadModel({
             model_path: repo.repo_id,
             hf_token: hfToken,
@@ -616,6 +624,17 @@ async function autoLoadSmallestModel(): Promise<{
       }
     }
 
+    // Cap also gates the default download so the total /api/inference/load
+    // budget across cached + fallback is MAX_AUTO_LOAD_ATTEMPTS, not +1.
+    if (loadAttempts >= MAX_AUTO_LOAD_ATTEMPTS) {
+      toast.dismiss(toastId);
+      return {
+        loaded: false,
+        blockedByTrustRemoteCode:
+          blockedByTrustRemoteCode && !hadNonTrustFailure,
+      };
+    }
+
     // No cached models found — try downloading a small default GGUF
     toast("Downloading a small model…", {
       id: toastId,
@@ -634,6 +653,7 @@ async function autoLoadSmallestModel(): Promise<{
         toast.dismiss(toastId);
         return { loaded: false, blockedByTrustRemoteCode };
       }
+      loadAttempts += 1;
       const loadResp = await loadModel({
         model_path: "unsloth/gemma-4-E2B-it-GGUF",
         hf_token: hfToken,