ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chat seq slider

Browse source
This commit is contained in:
Shine1i 2026-03-11 01:41:25 +01:00
commit 4a8a96b1af
5 changed files with 23 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

6
studio/backend/models/inference.py
View file

@ -17,7 +17,7 @@ class LoadRequest(BaseModel):
"""Request to load a model for inference"""
model_path: str = Field(..., description="Model identifier or local path")
hf_token: Optional[str] = Field(None, description="HuggingFace token for gated models")
max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length")
max_seq_length: int = Field(4096, ge=128, le=32768, description="Maximum sequence length")
load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization")
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')")
@ -67,7 +67,7 @@ class GenerateRequest(BaseModel):
temperature: float = Field(0.7, ge=0.0, le=2.0, description="Sampling temperature")
top_p: float = Field(0.9, ge=0.0, le=1.0, description="Top-p sampling")
top_k: int = Field(40, ge=-1, le=100, description="Top-k sampling")
max_new_tokens: int = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
max_new_tokens: int = Field(2048, ge=1, le=4096, description="Maximum tokens to generate")
repetition_penalty: float = Field(1.1, ge=1.0, le=2.0, description="Repetition penalty")
image_base64: Optional[str] = Field(None, description="Base64 encoded image for vision models")
@ -170,7 +170,7 @@ class ChatCompletionRequest(BaseModel):
stream: bool = Field(True, description="Whether to stream the response via SSE")
temperature: float = Field(0.7, ge=0.0, le=2.0)
top_p: float = Field(0.9, ge=0.0, le=1.0)
max_tokens: Optional[int] = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
max_tokens: Optional[int] = Field(2048, ge=1, le=4096, description="Maximum tokens to generate")
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
top_k: int = Field(40, ge=-1, le=100, description="[x-unsloth] Top-k sampling")