studio: optimize llama-server flags for single-user studio
Refactor command building (deduplicate HF/local paths) and add flags for better performance: - --parallel 1: studio is single-user, so only 1 inference slot is needed. The previous auto-detect picked 4 slots, wasting VRAM on 3 unused KV caches. - --flash-attn on: force flash attention for faster inference. Default is "auto" which may not always enable it. - --fit on: auto-adjust parameters to fit in available device memory. Already the default but now explicit. Also cleaned up the duplicated command building for HF vs local mode into a single block.
This commit is contained in:
parent
887e7a31c4
commit
80d84a5b5f
1 changed files with 13 additions and 23 deletions
|
|
@ -334,38 +334,28 @@ class LlamaCppBackend:
|
|||
)
|
||||
|
||||
logger.info(f"GGUF downloaded to: {local_path}")
|
||||
cmd = [
|
||||
binary,
|
||||
"-m",
|
||||
local_path,
|
||||
"--port",
|
||||
str(self._port),
|
||||
"-c",
|
||||
str(n_ctx),
|
||||
"-ngl",
|
||||
str(n_gpu_layers),
|
||||
]
|
||||
model_path = local_path
|
||||
elif gguf_path:
|
||||
if not Path(gguf_path).is_file():
|
||||
raise FileNotFoundError(f"GGUF file not found: {gguf_path}")
|
||||
cmd = [
|
||||
binary,
|
||||
"-m",
|
||||
gguf_path,
|
||||
"--port",
|
||||
str(self._port),
|
||||
"-c",
|
||||
str(n_ctx),
|
||||
"-ngl",
|
||||
str(n_gpu_layers),
|
||||
]
|
||||
model_path = gguf_path
|
||||
else:
|
||||
raise ValueError("Either gguf_path or hf_repo must be provided")
|
||||
|
||||
cmd = [
|
||||
binary,
|
||||
"-m", model_path,
|
||||
"--port", str(self._port),
|
||||
"-c", str(n_ctx),
|
||||
"-ngl", str(n_gpu_layers),
|
||||
"--parallel", "1", # Single-user studio, saves VRAM
|
||||
"--flash-attn", "on", # Force flash attention for speed
|
||||
"--fit", "on", # Auto-fit to available device memory
|
||||
]
|
||||
|
||||
if n_threads is not None:
|
||||
cmd.extend(["--threads", str(n_threads)])
|
||||
|
||||
# Append mmproj for local vision models
|
||||
if mmproj_path:
|
||||
if not Path(mmproj_path).is_file():
|
||||
logger.warning(f"mmproj file not found: {mmproj_path}")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue