diff --git a/studio/backend/assets/configs/full_finetune.yaml b/studio/backend/assets/configs/full_finetune.yaml index 7536ed1f11..e398515f61 100644 --- a/studio/backend/assets/configs/full_finetune.yaml +++ b/studio/backend/assets/configs/full_finetune.yaml @@ -10,13 +10,13 @@ training: load_in_4bit: false output_dir: outputs num_epochs: 1 - learning_rate: 0.0002 + learning_rate: 2e-5 batch_size: 1 gradient_accumulation_steps: 4 warmup_steps: 5 max_steps: 0 save_steps: 0 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/assets/configs/lora_text.yaml b/studio/backend/assets/configs/lora_text.yaml index 7101a00e85..9cb6b8c700 100644 --- a/studio/backend/assets/configs/lora_text.yaml +++ b/studio/backend/assets/configs/lora_text.yaml @@ -16,7 +16,7 @@ training: warmup_steps: 5 max_steps: 0 save_steps: 0 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/assets/configs/model_defaults/default.yaml b/studio/backend/assets/configs/model_defaults/default.yaml index d96e5077b2..12566019b8 100644 --- a/studio/backend/assets/configs/model_defaults/default.yaml +++ b/studio/backend/assets/configs/model_defaults/default.yaml @@ -6,13 +6,13 @@ training: max_seq_length: 2048 # num_epochs: 4 num_epochs: 0 - learning_rate: 5e-5 + learning_rate: 2e-4 batch_size: 2 gradient_accumulation_steps: 4 warmup_ratio: 0.1 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: true diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml index 1219648a8a..f7b49c75b7 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_Qwen3-Embedding-0.6B.yaml @@ -12,7 +12,7 @@ training: warmup_ratio: 0.03 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml index db742e11b5..be7da0f624 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_all-MiniLM-L6-v2.yaml @@ -11,7 +11,7 @@ training: warmup_ratio: 0.03 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml index 499c112929..d9e49bc0d5 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_bge-m3.yaml @@ -11,7 +11,7 @@ training: warmup_ratio: 0.03 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml index 016d284a16..c3422d399f 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_embeddinggemma-300m.yaml @@ -11,7 +11,7 @@ training: warmup_ratio: 0.03 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml b/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml index eefaecc8cd..529a56a527 100644 --- a/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml +++ b/studio/backend/assets/configs/model_defaults/embedding/unsloth_gte-modernbert-base.yaml @@ -11,7 +11,7 @@ training: warmup_ratio: 0.03 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml index c45b71b4ae..fa7bd8c1ea 100644 --- a/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml @@ -13,7 +13,7 @@ training: warmup_steps: 5 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: true diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml index f8f78f5edc..a4acbe9262 100644 --- a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml @@ -13,7 +13,7 @@ training: warmup_steps: 5 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: true diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml index 3938f10627..2bc3f6f871 100644 --- a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml @@ -13,7 +13,7 @@ training: warmup_steps: 0 max_steps: 30 save_steps: 30 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: true diff --git a/studio/backend/assets/configs/vision_lora.yaml b/studio/backend/assets/configs/vision_lora.yaml index 60641b16e8..063a970316 100644 --- a/studio/backend/assets/configs/vision_lora.yaml +++ b/studio/backend/assets/configs/vision_lora.yaml @@ -16,7 +16,7 @@ training: warmup_steps: 5 max_steps: 0 save_steps: 0 - weight_decay: 0.01 + weight_decay: 0.001 random_seed: 3407 packing: false train_on_completions: false diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.py index 9b4a14f09f..ab1825d94a 100644 --- a/studio/backend/core/training/trainer.py +++ b/studio/backend/core/training/trainer.py @@ -2658,14 +2658,14 @@ class UnslothTrainer: eval_steps: float = 0.00, output_dir: str | None = None, num_epochs: int = 3, - learning_rate: float = 5e-5, + learning_rate: float = 2e-4, batch_size: int = 2, gradient_accumulation_steps: int = 4, warmup_steps: int = None, warmup_ratio: float = None, max_steps: int = 0, save_steps: int = 0, - weight_decay: float = 0.01, + weight_decay: float = 0.001, random_seed: int = 3407, packing: bool = False, train_on_completions: bool = False, @@ -3034,7 +3034,7 @@ class UnslothTrainer: "fp16": not is_bfloat16_supported(), "bf16": is_bfloat16_supported(), "logging_steps": 1, - "weight_decay": training_args.get("weight_decay", 0.01), + "weight_decay": training_args.get("weight_decay", 0.001), "seed": training_args.get("random_seed", 3407), "output_dir": output_dir, "report_to": _build_report_targets(training_args), diff --git a/studio/backend/core/training/training.py b/studio/backend/core/training/training.py index 2c8f9a21db..f35c7e8ad3 100644 --- a/studio/backend/core/training/training.py +++ b/studio/backend/core/training/training.py @@ -160,7 +160,7 @@ class TrainingBackend: "warmup_ratio": kwargs.get("warmup_ratio"), "max_steps": kwargs.get("max_steps", 0), "save_steps": kwargs.get("save_steps", 0), - "weight_decay": kwargs.get("weight_decay", 0.01), + "weight_decay": kwargs.get("weight_decay", 0.001), "random_seed": kwargs.get("random_seed", 3407), "packing": kwargs.get("packing", False), "optim": kwargs.get("optim", "adamw_8bit"), diff --git a/studio/backend/core/training/worker.py b/studio/backend/core/training/worker.py index e68a6c7aee..699cfe74f7 100644 --- a/studio/backend/core/training/worker.py +++ b/studio/backend/core/training/worker.py @@ -795,7 +795,7 @@ def run_training_process( warmup_ratio = config.get("warmup_ratio"), max_steps = max_steps if max_steps and max_steps > 0 else 0, save_steps = save_steps if save_steps and save_steps > 0 else 0, - weight_decay = config.get("weight_decay", 0.01), + weight_decay = config.get("weight_decay", 0.001), random_seed = config.get("random_seed", 3407), packing = config.get("packing", False), train_on_completions = config.get("train_on_completions", False), @@ -1141,7 +1141,7 @@ def _run_embedding_training(event_queue: Any, stop_queue: Any, config: dict) -> "lr_scheduler_type": config.get("lr_scheduler_type", "linear"), "batch_sampler": BatchSamplers.NO_DUPLICATES, "optim": config.get("optim", "adamw_8bit"), - "weight_decay": config.get("weight_decay", 0.01), + "weight_decay": config.get("weight_decay", 0.001), "seed": config.get("random_seed", 3407), } diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py index eeb98c872e..07a306ca39 100644 --- a/studio/backend/models/training.py +++ b/studio/backend/models/training.py @@ -81,7 +81,7 @@ class TrainingStartRequest(BaseModel): warmup_ratio: Optional[float] = Field(None, description = "Warmup ratio") max_steps: Optional[int] = Field(None, description = "Maximum training steps") save_steps: int = Field(100, description = "Steps between checkpoints") - weight_decay: float = Field(0.01, description = "Weight decay") + weight_decay: float = Field(0.001, description = "Weight decay") random_seed: int = Field(42, description = "Random seed") packing: bool = Field(False, description = "Enable sequence packing") optim: str = Field("adamw_8bit", description = "Optimizer") diff --git a/studio/frontend/src/config/training.ts b/studio/frontend/src/config/training.ts index 677e91fdd7..42fccde552 100644 --- a/studio/frontend/src/config/training.ts +++ b/studio/frontend/src/config/training.ts @@ -90,10 +90,17 @@ export const LR_SCHEDULER_OPTIONS: ReadonlyArray<{ value: string; label: string { value: "cosine", label: "Cosine" }, ]; +/** + * Method-aware learning rate defaults. + * Backend mirrors these in the YAML configs under studio/backend/assets/configs/. + */ +export const LR_DEFAULT_LORA = 2e-4; +export const LR_DEFAULT_FULL = 2e-5; + export const DEFAULT_HYPERPARAMS = { epochs: 3, contextLength: 2048, - learningRate: 2e-4, + learningRate: LR_DEFAULT_LORA, optimizerType: "adamw_8bit", lrSchedulerType: "linear", loraRank: 16, @@ -102,7 +109,7 @@ export const DEFAULT_HYPERPARAMS = { loraVariant: "lora" as const, batchSize: 4, gradientAccumulation: 8, - weightDecay: 0.01, + weightDecay: 0.001, warmupSteps: 5, maxSteps: 60, saveSteps: 0, diff --git a/studio/frontend/src/features/training/stores/training-config-store.ts b/studio/frontend/src/features/training/stores/training-config-store.ts index c658f15016..8214b0eb2a 100644 --- a/studio/frontend/src/features/training/stores/training-config-store.ts +++ b/studio/frontend/src/features/training/stores/training-config-store.ts @@ -1,8 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-only // Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -import { DEFAULT_HYPERPARAMS, STEPS } from "@/config/training"; +import { DEFAULT_HYPERPARAMS, LR_DEFAULT_FULL, LR_DEFAULT_LORA, STEPS } from "@/config/training"; import { authFetch } from "@/features/auth"; +import { isAdapterMethod } from "@/types/training"; import type { ModelType, StepNumber, TrainingMethod } from "@/types/training"; import { create } from "zustand"; import { persist } from "zustand/middleware"; @@ -98,6 +99,15 @@ let _modelConfigController: AbortController | null = null; // since the last auto-set (model load or dataset change). let _trainOnCompletionsManuallySet = false; +// Track whether the user has manually edited the learning rate +// since the last model load. When false, switching training method +// auto-sets LR to 2e-4 (LoRA/QLoRA) or 2e-5 (full fine-tune). +let _learningRateManuallySet = false; + +// Stash the model-config-provided (YAML) learning rate so that +// setTrainingMethod can restore it when switching back from full to adapter. +let _yamlLearningRate: number | undefined = undefined; + const NON_PERSISTED_STATE_KEYS: ReadonlySet = new Set([ "modelType", "isCheckingVision", @@ -165,8 +175,22 @@ export const useTrainingConfigStore = create()( if (get().selectedModel !== modelName) return; _trainOnCompletionsManuallySet = false; + _learningRateManuallySet = false; + _yamlLearningRate = undefined; const patch = mapBackendModelConfigToTrainingPatch(modelDetails.config); + // If the model config provides a specific learning rate, treat + // it as authoritative so the async auto-select does not overwrite it. + const modelConfigHasLR = patch.learningRate !== undefined; + _yamlLearningRate = patch.learningRate; + + // YAML learning rates are tuned for adapter methods (LoRA/QLoRA). + // If the user is currently on full fine-tune, override with the + // full-finetune default instead of applying the YAML adapter LR. + if (modelConfigHasLR && !isAdapterMethod(get().trainingMethod)) { + patch.learningRate = LR_DEFAULT_FULL; + } + // If vision model + image dataset already known, override // trainOnCompletions to false regardless of backend default. if (modelDetails.is_vision && get().isDatasetImage === true) { @@ -174,11 +198,11 @@ export const useTrainingConfigStore = create()( } const isAudio = !!modelDetails.is_audio; - // Pure audio model → always uncheck trainOnCompletions. + // Pure audio model -> always uncheck trainOnCompletions. if (isAudio && !modelDetails.is_vision) { patch.trainOnCompletions = false; } - // Audio-capable vision model (e.g. gemma3n) + audio dataset → uncheck. + // Audio-capable vision model (e.g. gemma3n) + audio dataset -> uncheck. if (isAudio && modelDetails.is_vision && get().isDatasetAudio) { patch.trainOnCompletions = false; } @@ -197,7 +221,12 @@ export const useTrainingConfigStore = create()( void autoSelectTrainingMethod(modelSizeBytes, patch.contextLength ?? get().contextLength) .then((method) => { if (get().selectedModel !== modelName) return; - if (method) set({ trainingMethod: method }); + if (method) { + const lrPatch = !_learningRateManuallySet && !modelConfigHasLR + ? { learningRate: method === "full" ? LR_DEFAULT_FULL : LR_DEFAULT_LORA } + : {}; + set({ trainingMethod: method, ...lrPatch }); + } }); } @@ -366,7 +395,31 @@ export const useTrainingConfigStore = create()( if (state.modelDefaultsAppliedFor === state.selectedModel) return; void loadAndApplyModelDefaults(state.selectedModel); }, - setTrainingMethod: (trainingMethod) => set({ trainingMethod }), + setTrainingMethod: (trainingMethod) => { + if (_learningRateManuallySet) { + set({ trainingMethod }); + return; + } + + const prev = get().trainingMethod; + const wasAdapter = isAdapterMethod(prev); + const nowAdapter = isAdapterMethod(trainingMethod); + + // qlora <-> lora: same LR range, don't touch learning rate + if (wasAdapter && nowAdapter) { + set({ trainingMethod }); + return; + } + + // Category changed (adapter <-> full) + if (nowAdapter) { + // Switching TO adapter: restore YAML LR if available + set({ trainingMethod, learningRate: _yamlLearningRate ?? LR_DEFAULT_LORA }); + } else { + // Switching TO full: no YAML full-LR exists, use constant + set({ trainingMethod, learningRate: LR_DEFAULT_FULL }); + } + }, setHfToken: (hfToken) => set({ hfToken: hfToken.trim().replace(/^["']+|["']+$/g, "") }), setDatasetSource: (datasetSource) => set({ datasetSource }), @@ -509,7 +562,10 @@ export const useTrainingConfigStore = create()( }), setEpochs: (epochs) => set({ epochs }), setContextLength: (contextLength) => set({ contextLength }), - setLearningRate: (learningRate) => set({ learningRate }), + setLearningRate: (learningRate) => { + _learningRateManuallySet = true; + set({ learningRate }); + }, setOptimizerType: (optimizerType) => set({ optimizerType }), setLrSchedulerType: (lrSchedulerType) => set({ lrSchedulerType }), setLoraRank: (loraRank) => set({ loraRank }), @@ -548,7 +604,12 @@ export const useTrainingConfigStore = create()( set({ finetuneMLPModules }), setTargetModules: (targetModules) => set({ targetModules }), canProceed: () => canProceedForStep(get()), - reset: () => set(initialState), + reset: () => { + _trainOnCompletionsManuallySet = false; + _learningRateManuallySet = false; + _yamlLearningRate = undefined; + set(initialState); + }, resetToModelDefaults: () => { const { selectedModel } = get(); if (!selectedModel) return; @@ -557,13 +618,18 @@ export const useTrainingConfigStore = create()( }, applyConfigPatch: (config: BackendModelConfig) => { const patch = mapBackendModelConfigToTrainingPatch(config); + // Only clear the manual-edit flag when the config provides a LR, + // so unrelated config patches don't silently disarm the guard. + if (patch.learningRate !== undefined) { + _learningRateManuallySet = false; + } set(patch); }, }; }, { name: "unsloth_training_config_v1", - version: 8, + version: 9, migrate: (persisted, version) => { const s = persisted as Record; if (version < 2 && s.datasetSubset == null && s.datasetConfig != null) { @@ -593,6 +659,12 @@ export const useTrainingConfigStore = create()( s.datasetLabelMapping ??= {}; s.datasetAdvisorNotification ??= null; } + if (version < 9) { + // weight_decay default changed from 0.01 to 0.001. + if (s.weightDecay === 0.01) { + s.weightDecay = DEFAULT_HYPERPARAMS.weightDecay; + } + } return s as unknown as TrainingConfigStore; }, partialize: partializePersistedState,