From 5a38447b25da0b3faab644f7eb6c3f7b557f73e8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 16 Jun 2026 01:32:19 -0700 Subject: [PATCH] Studio: omit --threads when unset so llama.cpp picks physical cores (#5894) * Studio: omit --threads when unset so llama.cpp picks physical cores Studio passed --threads -1 when no thread count was set. The intent was physical cores, but an explicit --threads -1 makes llama.cpp's arg parser resolve it to hardware_concurrency() (every hyperthread), which contends on the memory bus and slows CPU and hybrid decode (a user saw about 60-75 fall to about 20-30 tok/s under CPU offload). Leaving --threads unset keeps n_threads at -1, which llama.cpp resolves to physical cores via common_cpu_get_num_math(). Omit the flag when unset; still pin it for an explicit override and the Windows full-offload OpenMP cap. * Studio: drop inherited LLAMA_ARG_THREADS when omitting --threads Omitting --threads relies on llama.cpp resolving physical cores via common_cpu_get_num_math. But the child inherits os.environ and llama.cpp also reads --threads from LLAMA_ARG_THREADS, which routes through the arg handler and maps <=0 to hardware_concurrency. So an ambient LLAMA_ARG_THREADS would silently override the physical-core default. Scrub it from the child env only when we omit the flag. --- studio/backend/core/inference/llama_cpp.py | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 59628c9e46..bd6bdad8c6 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -4166,19 +4166,21 @@ class LlamaCppBackend: threads_overridden = _extra_args_set_any_flag(extra_args, _THREAD_OVERRIDE_FLAGS) full_offload_tuning_active = fully_gpu_offloaded and not offload_overridden - # Pass --threads explicitly so we do not inherit llama-server - # defaults. Windows + full offload caps at 2 to stop OpenMP - # spin-wait burning CPU during GPU decode. User pass-through - # offload/thread flags keep last-wins semantics. #5692. + # Thread count: an unset --threads makes llama.cpp pick physical + # cores (common_cpu_get_num_math), but an explicit --threads -1 + # resolves to hardware_concurrency() (every hyperthread), which + # contends on the memory bus and slows CPU / hybrid decode. So + # omit the flag when unset and only pin it for an explicit + # override or the Windows full-offload OpenMP cap. Pass-through + # thread flags in extra_args still win (appended last). #5692 if ( sys.platform == "win32" and full_offload_tuning_active and not threads_overridden ): - threads_arg = 2 - else: - threads_arg = n_threads if n_threads is not None else -1 - cmd.extend(["--threads", str(threads_arg)]) + cmd.extend(["--threads", "2"]) + elif n_threads is not None and n_threads > 0: + cmd.extend(["--threads", str(n_threads)]) # Enable Jinja chat template rendering cmd.extend(["--jinja"]) @@ -4358,6 +4360,11 @@ class LlamaCppBackend: # Library paths so llama-server finds its shared libs and CUDA DLLs. env = self._llama_server_env_for_binary(binary) + # Omitting --threads relies on llama.cpp's physical-core default, so + # drop an inherited LLAMA_ARG_THREADS that would otherwise feed the + # arg handler and silently force hardware_concurrency(). #5692 + if "--threads" not in cmd: + env.pop("LLAMA_ARG_THREADS", None) # Windows + full offload: PASSIVE OMP + 2 threads stop # spin-wait burning CPU. CPU/partial offload keeps default