a1907fd4fe* feat(studio): add DoRA support to studio * fix: added use_dora fast encoder LoraConfig and gated use_dora on AdapterMethod * fix(studio) serverside normalization for use_dora=true - add note documenting use_dora is silently dropped on diffusion * fix: dora button disabled on mac, add preflight guard on GGUF lora export, mismatch now correctly falls through to existing error instead of silently no-opping * Studio: add dora to the WizardState LoRA variant union for consistency * Reject --use_dora on the MLX (Apple Silicon) CLI path --------- Co-authored-by: danielhanchen <unslothai@gmail.com>
50 lines
1.1 KiB
YAML
50 lines
1.1 KiB
YAML
# Model defaults for unsloth/gemma-3-4b-pt
|
|
# Based on Gemma3_(4B)-Vision.ipynb
|
|
# Also applies to: unsloth/gemma-3-4b-pt-unsloth-bnb-4bit, google/gemma-3-4b-pt, unsloth/gemma-3-4b-pt-bnb-4bit
|
|
# added inference parameters from unsloth guides
|
|
|
|
training:
|
|
max_seq_length: 2048
|
|
# num_epochs: 2
|
|
num_epochs: 0
|
|
learning_rate: 2e-4
|
|
batch_size: 1
|
|
gradient_accumulation_steps: 4
|
|
warmup_ratio: 0.03
|
|
max_steps: 30
|
|
save_steps: 30
|
|
weight_decay: 0.001
|
|
random_seed: 3407
|
|
packing: false
|
|
train_on_completions: true
|
|
gradient_checkpointing: true
|
|
optim: "adamw_torch_fused"
|
|
lr_scheduler_type: "cosine"
|
|
|
|
lora:
|
|
lora_r: 16
|
|
lora_alpha: 16
|
|
lora_dropout: 0.0
|
|
target_modules:
|
|
- "all-linear"
|
|
use_rslora: false
|
|
use_loftq: false
|
|
use_dora: false
|
|
finetune_vision_layers: true
|
|
finetune_language_layers: true
|
|
finetune_attention_modules: true
|
|
finetune_mlp_modules: true
|
|
|
|
logging:
|
|
enable_wandb: false
|
|
wandb_project: "llm-finetuning"
|
|
enable_tensorboard: false
|
|
tensorboard_dir: "runs"
|
|
log_frequency: 10
|
|
|
|
inference:
|
|
temperature: 1.0
|
|
top_k: 64
|
|
top_p: 0.95
|
|
min_p: 0.0
|
|
|