feat: support disabling top-k sampling with -1 and standardize normalization logic
- Updated top-k parameter range to accept -1 in models and frontend. - Added utility to normalize top-k for backend compatibility.
This commit is contained in:
parent
43d84d7143
commit
0db7da96cc
3 changed files with 9 additions and 3 deletions
|
|
@ -45,6 +45,11 @@ class InferenceBackend:
|
|||
|
||||
logger.info(f"InferenceBackend initialized on {self.device}")
|
||||
|
||||
@staticmethod
|
||||
def _normalize_top_k(top_k: int) -> int:
|
||||
# API supports -1 as "disable top-k"; transformers expects 0 to disable.
|
||||
return 0 if top_k < 0 else top_k
|
||||
|
||||
def load_model(self,
|
||||
config: ModelConfig,
|
||||
max_seq_length: int = 2048,
|
||||
|
|
@ -560,6 +565,7 @@ class InferenceBackend:
|
|||
model_info = self.models[self.active_model_name]
|
||||
is_vision = model_info.get("is_vision", False)
|
||||
tokenizer = model_info.get("tokenizer") or model_info.get("processor")
|
||||
top_k = self._normalize_top_k(top_k)
|
||||
|
||||
if is_vision:
|
||||
# Vision model generation
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ class GenerateRequest(BaseModel):
|
|||
system_prompt: str = Field("You are a helpful AI assistant.", description="System prompt")
|
||||
temperature: float = Field(0.7, ge=0.0, le=2.0, description="Sampling temperature")
|
||||
top_p: float = Field(0.9, ge=0.0, le=1.0, description="Top-p sampling")
|
||||
top_k: int = Field(40, ge=1, le=100, description="Top-k sampling")
|
||||
top_k: int = Field(40, ge=-1, le=100, description="Top-k sampling")
|
||||
max_new_tokens: int = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
|
||||
repetition_penalty: float = Field(1.1, ge=1.0, le=2.0, description="Repetition penalty")
|
||||
image_base64: Optional[str] = Field(None, description="Base64 encoded image for vision models")
|
||||
|
|
@ -128,7 +128,7 @@ class ChatCompletionRequest(BaseModel):
|
|||
max_tokens: Optional[int] = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
|
||||
|
||||
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
|
||||
top_k: int = Field(40, ge=1, le=100, description="[x-unsloth] Top-k sampling")
|
||||
top_k: int = Field(40, ge=-1, le=100, description="[x-unsloth] Top-k sampling")
|
||||
min_p: float = Field(0.0, ge=0.0, le=1.0, description="[x-unsloth] Min-p sampling threshold")
|
||||
repetition_penalty: float = Field(1.1, ge=1.0, le=2.0, description="[x-unsloth] Repetition penalty")
|
||||
image_base64: Optional[str] = Field(None, description="[x-unsloth] Base64-encoded image for vision models")
|
||||
|
|
|
|||
|
|
@ -295,7 +295,7 @@ export function ChatSettingsPanel({
|
|||
<ParamSlider
|
||||
label="Top K"
|
||||
value={params.topK}
|
||||
min={0}
|
||||
min={-1}
|
||||
max={100}
|
||||
step={1}
|
||||
onChange={set("topK")}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue