47 lines
1,015 B
YAML
47 lines
1,015 B
YAML
# Model defaults for unsloth/Qwen2.5-1.5B-Instruct
|
|
# Based on nemo_gym_sudoku.ipynb
|
|
# Also applies to: unsloth/Qwen2.5-1.5B-Instruct-unsloth-bnb-4bit, Qwen/Qwen2.5-1.5B-Instruct, unsloth/Qwen2.5-1.5B-Instruct-bnb-4bit
|
|
|
|
training:
|
|
trust_remote_code: false
|
|
max_seq_length: 4096
|
|
# num_epochs: 4
|
|
num_epochs: 0
|
|
learning_rate: 1e-5
|
|
batch_size: 1
|
|
gradient_accumulation_steps: 64
|
|
warmup_ratio: 0.1
|
|
max_steps: 30
|
|
save_steps: 30
|
|
weight_decay: 0.001
|
|
random_seed: 42
|
|
packing: false
|
|
train_on_completions: true
|
|
gradient_checkpointing: "unsloth"
|
|
optim: "adamw_8bit"
|
|
lr_scheduler_type: "linear"
|
|
|
|
lora:
|
|
lora_r: 4
|
|
lora_alpha: 8
|
|
lora_dropout: 0.0
|
|
target_modules:
|
|
- "q_proj"
|
|
- "k_proj"
|
|
- "v_proj"
|
|
- "o_proj"
|
|
- "gate_proj"
|
|
- "up_proj"
|
|
- "down_proj"
|
|
use_rslora: false
|
|
use_loftq: false
|
|
|
|
logging:
|
|
enable_wandb: false
|
|
wandb_project: "llm-finetuning"
|
|
enable_tensorboard: false
|
|
tensorboard_dir: "runs"
|
|
log_frequency: 10
|
|
|
|
inference:
|
|
trust_remote_code: false
|