From 5ce4ab4d54d22af53581b0acf512dfee8d5ce987 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 19 May 2026 03:16:05 -0700 Subject: [PATCH] studio: emit one comma-chained --spec-type for CPU/Mac MTP path (#5575) * studio: emit one comma-chained --spec-type for CPU/Mac MTP path llama-server takes a single --spec-type whose value may be comma-separated to chain implementations (e.g. ngram-mod,draft-mtp). The CPU/Mac MTP branch in LlamaCppBackend.load_model was passing --spec-type twice in the same invocation, which is not the documented chaining mechanism and silently drops one of the two specs depending on llama.cpp's argv handling. Collapse the pair to --spec-type ngram-mod,{mtp_token} and update the stale _extra_args_set_spec_type docstring that claimed llama-server accumulates repeated --spec-type. Update the matching pass-through fixture in test_llama_server_args.py. * studio: align MTP ngram-mod knobs with llama.cpp upstream defaults Two correctness fixes against the llama.cpp server README: 1. The CPU/Mac comma-chained branch was emitting --spec-ngram-mod-n-max 6 with --spec-ngram-mod-n-min 48, which is nonsensical (min > max). Per the upstream default the value is 64. 2. The standalone ngram-mod branch was emitting --spec-ngram-size-n, --draft-min, --draft-max. llama.cpp removed those arg aliases for ngram-mod (they live only on the ngram-simple / map families now); the correct knobs are --spec-ngram-mod-n-match / n-min / n-max. Also refresh the inline comment block to point at the server README rather than the older docs/speculative.md draft- aliases. --- studio/backend/core/inference/llama_cpp.py | 31 +++++++++++-------- .../backend/tests/test_llama_server_args.py | 6 ++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 21f2fe71b5..95a8c26a3a 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -471,8 +471,9 @@ def _is_mtp_model_name( def _extra_args_set_spec_type(extra_args: Optional[Iterable[str]]) -> bool: - """User passed --spec-type / --spec-default? llama-server accumulates - repeated --spec-type, so we suppress auto-emit when this is true.""" + """User passed --spec-type / --spec-default? llama-server takes a + single --spec-type (comma-separated to chain), so suppress + auto-emit when this is true.""" if not extra_args: return False for raw in extra_args: @@ -2631,10 +2632,10 @@ class LlamaCppBackend: # Qwen3-235B offloaded | 12 t/s | 21 t/s | 1.8x # gpt-oss-120b repeat (92% accept)| 181 t/s | 814 t/s | 4.5x # - # Params from llama.cpp docs (docs/speculative.md): - # --spec-ngram-size-n 24 (small n not recommended) - # --draft-min 48 --draft-max 64 (MoEs need long drafts; - # dense models can reduce these) + # Params from llama.cpp server README: + # --spec-ngram-mod-n-match 24 (lookup length) + # --spec-ngram-mod-n-min 48 --spec-ngram-mod-n-max 64 + # (MoEs need long drafts; dense models can reduce these) # ref: https://github.com/ggml-org/llama.cpp/blob/master/docs/speculative.md # ref: https://github.com/ggml-org/llama.cpp/pull/19164 # ref: https://github.com/ggml-org/llama.cpp/pull/18471 @@ -2692,20 +2693,22 @@ class LlamaCppBackend: ] ) else: + # CPU/Mac: chain ngram-mod + MTP in one + # comma-separated --spec-type (not repeated). + # ngram-mod knobs match llama.cpp defaults + # (n-match 24, n-min 48, n-max 64). cmd.extend( [ "--spec-type", - mtp_token, + f"ngram-mod,{mtp_token}", "--spec-draft-n-max", "3", - "--spec-type", - "ngram-mod", "--spec-ngram-mod-n-match", "24", "--spec-ngram-mod-n-min", "48", "--spec-ngram-mod-n-max", - "6", + "64", ] ) self._speculative_type = "draft-mtp" @@ -2715,13 +2718,15 @@ class LlamaCppBackend: elif normalized_spec in _valid_spec_types: cmd.extend(["--spec-type", normalized_spec]) if normalized_spec == "ngram-mod": + # llama.cpp defaults; legacy --spec-ngram-size-n + # / --draft-{min,max} were removed for ngram-mod. cmd.extend( [ - "--spec-ngram-size-n", + "--spec-ngram-mod-n-match", "24", - "--draft-min", + "--spec-ngram-mod-n-min", "48", - "--draft-max", + "--spec-ngram-mod-n-max", "64", ] ) diff --git a/studio/backend/tests/test_llama_server_args.py b/studio/backend/tests/test_llama_server_args.py index f4dabfcf08..68a1c870fb 100644 --- a/studio/backend/tests/test_llama_server_args.py +++ b/studio/backend/tests/test_llama_server_args.py @@ -47,17 +47,15 @@ from core.inference.llama_server_args import ( ["--spec-type", "draft-mtp", "--spec-draft-n-max", "6"], [ "--spec-type", - "draft-mtp", + "ngram-mod,draft-mtp", "--spec-draft-n-max", "3", - "--spec-type", - "ngram-mod", "--spec-ngram-mod-n-match", "24", "--spec-ngram-mod-n-min", "48", "--spec-ngram-mod-n-max", - "6", + "64", ], # Reasoning controls ["--reasoning-format", "deepseek"],