diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 37f342f48c..573b6e9a06 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -17,7 +17,7 @@ class LoadRequest(BaseModel):
"""Request to load a model for inference"""
model_path: str = Field(..., description="Model identifier or local path")
hf_token: Optional[str] = Field(None, description="HuggingFace token for gated models")
- max_seq_length: int = Field(2048, ge=128, le=32768, description="Maximum sequence length")
+ max_seq_length: int = Field(4096, ge=128, le=32768, description="Maximum sequence length")
load_in_4bit: bool = Field(True, description="Load model in 4-bit quantization")
is_lora: bool = Field(False, description="Whether this is a LoRA adapter")
gguf_variant: Optional[str] = Field(None, description="GGUF quantization variant (e.g. 'Q4_K_M')")
@@ -67,7 +67,7 @@ class GenerateRequest(BaseModel):
temperature: float = Field(0.7, ge=0.0, le=2.0, description="Sampling temperature")
top_p: float = Field(0.9, ge=0.0, le=1.0, description="Top-p sampling")
top_k: int = Field(40, ge=-1, le=100, description="Top-k sampling")
- max_new_tokens: int = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
+ max_new_tokens: int = Field(2048, ge=1, le=4096, description="Maximum tokens to generate")
repetition_penalty: float = Field(1.1, ge=1.0, le=2.0, description="Repetition penalty")
image_base64: Optional[str] = Field(None, description="Base64 encoded image for vision models")
@@ -170,7 +170,7 @@ class ChatCompletionRequest(BaseModel):
stream: bool = Field(True, description="Whether to stream the response via SSE")
temperature: float = Field(0.7, ge=0.0, le=2.0)
top_p: float = Field(0.9, ge=0.0, le=1.0)
- max_tokens: Optional[int] = Field(512, ge=1, le=4096, description="Maximum tokens to generate")
+ max_tokens: Optional[int] = Field(2048, ge=1, le=4096, description="Maximum tokens to generate")
# ── Unsloth extensions (ignored by standard OpenAI clients) ──
top_k: int = Field(40, ge=-1, le=100, description="[x-unsloth] Top-k sampling")
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 387268edbb..1f72f0642f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -724,7 +724,7 @@ async def openai_chat_completions(
top_p=payload.top_p,
top_k=payload.top_k,
min_p=payload.min_p,
- max_new_tokens=payload.max_tokens or 512,
+ max_new_tokens=payload.max_tokens or 2048,
repetition_penalty=payload.repetition_penalty,
cancel_event=cancel_event,
)
@@ -815,7 +815,7 @@ async def openai_chat_completions(
top_p=payload.top_p,
top_k=payload.top_k,
min_p=payload.min_p,
- max_tokens=payload.max_tokens or 512,
+ max_tokens=payload.max_tokens or 2048,
repetition_penalty=payload.repetition_penalty,
cancel_event=cancel_event,
)
@@ -946,7 +946,7 @@ async def openai_chat_completions(
top_p=payload.top_p,
top_k=payload.top_k,
min_p=payload.min_p,
- max_new_tokens=payload.max_tokens or 512,
+ max_new_tokens=payload.max_tokens or 2048,
repetition_penalty=payload.repetition_penalty,
)
diff --git a/studio/frontend/src/features/chat/chat-settings-sheet.tsx b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
index 62ec2fcf8f..32a76c084c 100644
--- a/studio/frontend/src/features/chat/chat-settings-sheet.tsx
+++ b/studio/frontend/src/features/chat/chat-settings-sheet.tsx
@@ -322,11 +322,19 @@ export function ChatSettingsPanel({
step={0.05}
onChange={set("repetitionPenalty")}
/>
+
diff --git a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
index da7d9258f7..61edf8e44b 100644
--- a/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
+++ b/studio/frontend/src/features/chat/hooks/use-chat-model-runtime.ts
@@ -19,8 +19,6 @@ import type {
InferenceParams,
} from "../types/runtime";
-const DEFAULT_MODEL_MAX_SEQ_LENGTH = 2048;
-
type SelectedModelInput = {
id: string;
isLora?: boolean;
@@ -222,13 +220,15 @@ export function useChatModelRuntime() {
let previousWasUnloaded = false;
const currentCheckpoint =
useChatRuntimeStore.getState().params.checkpoint;
+ const paramsBeforeLoad = useChatRuntimeStore.getState().params;
+ const maxSeqLength = paramsBeforeLoad.maxSeqLength;
try {
// Lightweight pre-flight validation: avoid unloading a working model
// if the new identifier is clearly invalid (e.g. bad HF id / path).
await validateModel({
model_path: modelId,
hf_token: null,
- max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
+ max_seq_length: maxSeqLength,
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
@@ -239,11 +239,10 @@ export function useChatModelRuntime() {
previousWasUnloaded = true;
}
- const paramsBeforeLoad = useChatRuntimeStore.getState().params;
const loadResponse = await loadModel({
model_path: modelId,
hf_token: null,
- max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
+ max_seq_length: maxSeqLength,
load_in_4bit: true,
is_lora: isLora,
gguf_variant: ggufVariant ?? null,
@@ -262,7 +261,7 @@ export function useChatModelRuntime() {
await loadModel({
model_path: previousCheckpoint,
hf_token: null,
- max_seq_length: DEFAULT_MODEL_MAX_SEQ_LENGTH,
+ max_seq_length: maxSeqLength,
load_in_4bit: true,
is_lora: previousIsLora,
gguf_variant: previousVariant,
diff --git a/studio/frontend/src/features/chat/types/runtime.ts b/studio/frontend/src/features/chat/types/runtime.ts
index b94dce994f..bab9aca64a 100644
--- a/studio/frontend/src/features/chat/types/runtime.ts
+++ b/studio/frontend/src/features/chat/types/runtime.ts
@@ -7,6 +7,7 @@ export interface InferenceParams {
topK: number;
minP: number;
repetitionPenalty: number;
+ maxSeqLength: number;
maxTokens: number;
systemPrompt: string;
checkpoint: string;
@@ -20,7 +21,8 @@ export const DEFAULT_INFERENCE_PARAMS: InferenceParams = {
topK: 50,
minP: 0.01,
repetitionPenalty: 1.1,
- maxTokens: 4092,
+ maxSeqLength: 4096,
+ maxTokens: 2048,
systemPrompt: "",
checkpoint: "",
trustRemoteCode: false,