studio: optimize llama-server flags for single-user studio

Refactor command building (deduplicate HF/local paths) and add
flags for better performance:

- --parallel 1: studio is single-user, so only 1 inference slot
  is needed. The previous auto-detect picked 4 slots, wasting
  VRAM on 3 unused KV caches.
- --flash-attn on: force flash attention for faster inference.
  Default is "auto" which may not always enable it.
- --fit on: auto-adjust parameters to fit in available device
  memory. Already the default but now explicit.

Also cleaned up the duplicated command building for HF vs local
mode into a single block.
This commit is contained in:
Daniel Han 2026-03-14 08:28:19 +00:00
commit 80d84a5b5f

View file

@ -334,38 +334,28 @@ class LlamaCppBackend:
)
logger.info(f"GGUF downloaded to: {local_path}")
cmd = [
binary,
"-m",
local_path,
"--port",
str(self._port),
"-c",
str(n_ctx),
"-ngl",
str(n_gpu_layers),
]
model_path = local_path
elif gguf_path:
if not Path(gguf_path).is_file():
raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
cmd = [
binary,
"-m",
gguf_path,
"--port",
str(self._port),
"-c",
str(n_ctx),
"-ngl",
str(n_gpu_layers),
]
model_path = gguf_path
else:
raise ValueError("Either gguf_path or hf_repo must be provided")
cmd = [
binary,
"-m", model_path,
"--port", str(self._port),
"-c", str(n_ctx),
"-ngl", str(n_gpu_layers),
"--parallel", "1", # Single-user studio, saves VRAM
"--flash-attn", "on", # Force flash attention for speed
"--fit", "on", # Auto-fit to available device memory
]
if n_threads is not None:
cmd.extend(["--threads", str(n_threads)])
# Append mmproj for local vision models
if mmproj_path:
if not Path(mmproj_path).is_file():
logger.warning(f"mmproj file not found: {mmproj_path}")