ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #364 from unslothai/feature/chat-seq-slider

Browse source 
chat seq slider
This commit is contained in:
Wasim Yousef Said 2026-03-11 01:56:48 +01:00 • committed by GitHub
commit 739838bd48
5 changed files with 23 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

6
studio/backend/models/inference.py
View file

@ -17,7 +17,7 @@ class LoadRequest(BaseModel):
"""Request to load a model for inference"""
model_path: str = Field(..., description="Model identifier or local path")
hf_token: Optional[str] = Field(None, description="HuggingFace token for gated models")
max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length")
max_seq_length: int = Field(4096, ge=128, le=32768, description="Maximum sequence length")
load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization")
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')")
@ -67,7 +67,7 @@ class GenerateRequest(BaseModel):
temperature: float = Field(0.7, ge=0.0, le=2.0, description="Sampling temperature")
top_p: float = Field(0.9, ge=0.0, le=1.0, description="Top-p sampling")
top_k: int = Field(40, ge=-1, le=100, description="Top-k sampling")
max_new_tokens: int = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
max_new_tokens: int = Field(2048, ge=1, le=4096, description="Maximum tokens to generate")
repetition_penalty: float = Field(1.1, ge=1.0, le=2.0, description="Repetition penalty")
image_base64: Optional[str] = Field(None, description="Base64 encoded image for vision models")
@ -170,7 +170,7 @@ class ChatCompletionRequest(BaseModel):
stream: bool = Field(True, description="Whether to stream the response via SSE")
temperature: float = Field(0.7, ge=0.0, le=2.0)
top_p: float = Field(0.9, ge=0.0, le=1.0)
max_tokens: Optional[int] = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
max_tokens: Optional[int] = Field(2048, ge=1, le=4096, description="Maximum tokens to generate")
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
top_k: int = Field(40, ge=-1, le=100, description="[x-unsloth] Top-k sampling")

6
studio/backend/routes/inference.py
View file

@ -724,7 +724,7 @@ async def openai_chat_completions(
top_p=payload.top_p,
top_k=payload.top_k,
min_p=payload.min_p,
max_new_tokens=payload.max_tokens or 512,
max_new_tokens=payload.max_tokens or 2048,
repetition_penalty=payload.repetition_penalty,
cancel_event=cancel_event,
)
@ -815,7 +815,7 @@ async def openai_chat_completions(
top_p=payload.top_p,
top_k=payload.top_k,
min_p=payload.min_p,
max_tokens=payload.max_tokens or 512,
max_tokens=payload.max_tokens or 2048,
repetition_penalty=payload.repetition_penalty,
cancel_event=cancel_event,
)
@ -946,7 +946,7 @@ async def openai_chat_completions(
top_p=payload.top_p,
top_k=payload.top_k,
min_p=payload.min_p,
max_new_tokens=payload.max_tokens or 512,
max_new_tokens=payload.max_tokens or 2048,
repetition_penalty=payload.repetition_penalty,
)

10
studio/frontend/src/features/chat/chat-settings-sheet.tsx
View file

@ -322,11 +322,19 @@ export function ChatSettingsPanel({
step={0.05}
onChange={set("repetitionPenalty")}
/>
<ParamSlider
label="Max Seq Length"
value={params.maxSeqLength}
min={128}
max={32768}
step={128}
onChange={set("maxSeqLength")}
/>
<ParamSlider
label="Max Tokens"
value={params.maxTokens}
min={64}
max={4092}
max={4096}
step={64}
onChange={set("maxTokens")}
/>

11
studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
View file

@ -19,8 +19,6 @@ import type {
InferenceParams,
} from "../types/runtime";
const DEFAULT_MODEL_MAX_SEQ_LENGTH = 2048;
type SelectedModelInput = {
id: string;
isLora?: boolean;
@ -222,13 +220,15 @@ export function useChatModelRuntime() {
let previousWasUnloaded = false;
const currentCheckpoint =
useChatRuntimeStore.getState().params.checkpoint;
const paramsBeforeLoad = useChatRuntimeStore.getState().params;
const maxSeqLength = paramsBeforeLoad.maxSeqLength;
try {
// Lightweight pre-flight validation: avoid unloading a working model
// if the new identifier is clearly invalid (e.g. bad HF id / path).
await validateModel({
model_path: modelId,
hf_token: null,
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
max_seq_length: maxSeqLength,
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
@ -239,11 +239,10 @@ export function useChatModelRuntime() {
previousWasUnloaded = true;
}
const paramsBeforeLoad = useChatRuntimeStore.getState().params;
const loadResponse = await loadModel({
model_path: modelId,
hf_token: null,
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
max_seq_length: maxSeqLength,
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
@ -262,7 +261,7 @@ export function useChatModelRuntime() {
await loadModel({
model_path: previousCheckpoint,
hf_token: null,
max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
max_seq_length: maxSeqLength,
load_in_4bit: true,
is_lora: previousIsLora,
gguf_variant: previousVariant,

4
studio/frontend/src/features/chat/types/runtime.ts
View file

@ -7,6 +7,7 @@ export interface InferenceParams {
topK: number;
minP: number;
repetitionPenalty: number;
maxSeqLength: number;
maxTokens: number;
systemPrompt: string;
checkpoint: string;
@ -20,7 +21,8 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = {
topK: 50,
minP: 0.01,
repetitionPenalty: 1.1,
maxTokens: 4092,
maxSeqLength: 4096,
maxTokens: 2048,
systemPrompt: "",
checkpoint: "",
trustRemoteCode: false,