From e62085a3d6a77ffbc0ae796836dd55e36769195c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 26 Mar 2026 20:20:53 -0700 Subject: [PATCH] Fix repetition_penalty default causing 24% TPS drop in GGUF inference (#4634) The ChatCompletionRequest Pydantic model defaulted repetition_penalty to 1.1 when clients omitted the field. This silently forced llama-server to perform per-token repetition scanning, dropping streaming throughput from ~225 TPS to ~172 TPS (a 24% penalty). The Studio frontend always sends repetition_penalty=1.0 explicitly, so UI users were unaffected. But any API client hitting /v1/chat/completions without setting the field (curl, third-party integrations, Open WebUI, etc.) would get the slow path. Benchmarked on Qwen3.5-4B Q4_K_XL, GPU 0: - repeat_penalty=1.0: 225.2 TPS - repeat_penalty=1.1: 172.7 TPS (24% slower) - LM Studio (which applies rp internally): 170.8 TPS This aligns the Pydantic default with the frontend default (1.0), generate_chat_completion's function signature default (1.0), and llama-server's own default (1.0). --- studio/backend/models/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index b4e496b051..36395af7bd 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -291,7 +291,7 @@ class ChatCompletionRequest(BaseModel): 0.01, ge = 0.0, le = 1.0, description = "[x-unsloth] Min-p sampling threshold" ) repetition_penalty: float = Field( - 1.1, ge = 1.0, le = 2.0, description = "[x-unsloth] Repetition penalty" + 1.0, ge = 1.0, le = 2.0, description = "[x-unsloth] Repetition penalty" ) image_base64: Optional[str] = Field( None, description = "[x-unsloth] Base64-encoded image for vision models"