From 80d84a5b5f8fc1533e4e39bdbd871f7de76d14a1 Mon Sep 17 00:00:00 2001
From: Daniel Han <danielhanchen@gmail.com>
Date: Sat, 14 Mar 2026 08:28:19 +0000
Subject: [PATCH] studio: optimize llama-server flags for single-user studio

Refactor command building (deduplicate HF/local paths) and add
flags for better performance:

- --parallel 1: studio is single-user, so only 1 inference slot
  is needed. The previous auto-detect picked 4 slots, wasting
  VRAM on 3 unused KV caches.
- --flash-attn on: force flash attention for faster inference.
  Default is "auto" which may not always enable it.
- --fit on: auto-adjust parameters to fit in available device
  memory. Already the default but now explicit.

Also cleaned up the duplicated command building for HF vs local
mode into a single block.
---
 studio/backend/core/inference/llama_cpp.py | 36 ++++++++--------------
 1 file changed, 13 insertions(+), 23 deletions(-)

diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 1d3934af8c..063a656ecb 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -334,38 +334,28 @@ class LlamaCppBackend:
                     )
 
                 logger.info(f"GGUF downloaded to: {local_path}")
-                cmd = [
-                    binary,
-                    "-m",
-                    local_path,
-                    "--port",
-                    str(self._port),
-                    "-c",
-                    str(n_ctx),
-                    "-ngl",
-                    str(n_gpu_layers),
-                ]
+                model_path = local_path
             elif gguf_path:
                 if not Path(gguf_path).is_file():
                     raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
-                cmd = [
-                    binary,
-                    "-m",
-                    gguf_path,
-                    "--port",
-                    str(self._port),
-                    "-c",
-                    str(n_ctx),
-                    "-ngl",
-                    str(n_gpu_layers),
-                ]
+                model_path = gguf_path
             else:
                 raise ValueError("Either gguf_path or hf_repo must be provided")
 
+            cmd = [
+                binary,
+                "-m", model_path,
+                "--port", str(self._port),
+                "-c", str(n_ctx),
+                "-ngl", str(n_gpu_layers),
+                "--parallel", "1",       # Single-user studio, saves VRAM
+                "--flash-attn", "on",    # Force flash attention for speed
+                "--fit", "on",           # Auto-fit to available device memory
+            ]
+
             if n_threads is not None:
                 cmd.extend(["--threads", str(n_threads)])
 
-            # Append mmproj for local vision models
             if mmproj_path:
                 if not Path(mmproj_path).is_file():
                     logger.warning(f"mmproj file not found: {mmproj_path}")