feat: support disabling top-k sampling with -1 and standardize normalization logic

- Updated top-k parameter range to accept -1 in models and frontend.
- Added utility to normalize top-k for backend compatibility.
This commit is contained in:
Shine1i 2026-02-16 21:33:24 +01:00
commit 0db7da96cc
3 changed files with 9 additions and 3 deletions

View file

@ -45,6 +45,11 @@ class InferenceBackend:
logger.info(f"InferenceBackend initialized on {self.device}")
@staticmethod
def _normalize_top_k(top_k: int) -> int:
# API supports -1 as "disable top-k"; transformers expects 0 to disable.
return 0 if top_k < 0 else top_k
def load_model(self,
config: ModelConfig,
max_seq_length: int = 2048,
@ -560,6 +565,7 @@ class InferenceBackend:
model_info = self.models[self.active_model_name]
is_vision = model_info.get("is_vision", False)
tokenizer = model_info.get("tokenizer") or model_info.get("processor")
top_k = self._normalize_top_k(top_k)
if is_vision:
# Vision model generation

View file

@ -30,7 +30,7 @@ class GenerateRequest(BaseModel):
system_prompt: str = Field("You are a helpful AI assistant.", description="System prompt")
temperature: float = Field(0.7, ge=0.0, le=2.0, description="Sampling temperature")
top_p: float = Field(0.9, ge=0.0, le=1.0, description="Top-p sampling")
top_k: int = Field(40, ge=1, le=100, description="Top-k sampling")
top_k: int = Field(40, ge=-1, le=100, description="Top-k sampling")
max_new_tokens: int = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
repetition_penalty: float = Field(1.1, ge=1.0, le=2.0, description="Repetition penalty")
image_base64: Optional[str] = Field(None, description="Base64 encoded image for vision models")
@ -128,7 +128,7 @@ class ChatCompletionRequest(BaseModel):
max_tokens: Optional[int] = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
top_k: int = Field(40, ge=1, le=100, description="[x-unsloth] Top-k sampling")
top_k: int = Field(40, ge=-1, le=100, description="[x-unsloth] Top-k sampling")
min_p: float = Field(0.0, ge=0.0, le=1.0, description="[x-unsloth] Min-p sampling threshold")
repetition_penalty: float = Field(1.1, ge=1.0, le=2.0, description="[x-unsloth] Repetition penalty")
image_base64: Optional[str] = Field(None, description="[x-unsloth] Base64-encoded image for vision models")

View file

@ -295,7 +295,7 @@ export function ChatSettingsPanel({
<ParamSlider
label="Top K"
value={params.topK}
min={0}
min={-1}
max={100}
step={1}
onChange={set("topK")}