42 lines
889 B
YAML
42 lines
889 B
YAML
model: unsloth/Qwen2.5-0.5B
|
|
|
|
data:
|
|
dataset: tatsu-lab/alpaca
|
|
format_type: auto
|
|
|
|
training:
|
|
training_type: lora
|
|
max_seq_length: 2048
|
|
load_in_4bit: true
|
|
output_dir: outputs
|
|
num_epochs: 1
|
|
learning_rate: 0.0002
|
|
batch_size: 2
|
|
gradient_accumulation_steps: 4
|
|
warmup_steps: 5
|
|
max_steps: 0
|
|
save_steps: 0
|
|
weight_decay: 0.01
|
|
random_seed: 3407
|
|
packing: false
|
|
train_on_completions: false
|
|
gradient_checkpointing: "unsloth"
|
|
|
|
lora:
|
|
lora_r: 64
|
|
lora_alpha: 16
|
|
lora_dropout: 0.0
|
|
target_modules: "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
|
|
vision_all_linear: false
|
|
use_rslora: false
|
|
use_loftq: false
|
|
finetune_vision_layers: true
|
|
finetune_language_layers: true
|
|
finetune_attention_modules: true
|
|
finetune_mlp_modules: true
|
|
|
|
logging:
|
|
enable_wandb: false
|
|
wandb_project: unsloth-training
|
|
enable_tensorboard: false
|
|
tensorboard_dir: runs
|