diff --git a/studio/backend/assets/configs/full_finetune.yaml b/studio/backend/assets/configs/full_finetune.yaml new file mode 100644 index 0000000000..7536ed1f11 --- /dev/null +++ b/studio/backend/assets/configs/full_finetune.yaml @@ -0,0 +1,42 @@ +model: unsloth/Qwen2.5-0.5B + +data: + dataset: tatsu-lab/alpaca + format_type: auto + +training: + training_type: full + max_seq_length: 2048 + load_in_4bit: false + output_dir: outputs + num_epochs: 1 + learning_rate: 0.0002 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.01 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + +lora: + lora_r: 64 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: "" + vision_all_linear: false + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: unsloth-training + enable_tensorboard: false + tensorboard_dir: runs diff --git a/studio/backend/assets/configs/lora_text.yaml b/studio/backend/assets/configs/lora_text.yaml new file mode 100644 index 0000000000..7101a00e85 --- /dev/null +++ b/studio/backend/assets/configs/lora_text.yaml @@ -0,0 +1,42 @@ +model: unsloth/Qwen2.5-0.5B + +data: + dataset: tatsu-lab/alpaca + format_type: auto + +training: + training_type: lora + max_seq_length: 2048 + load_in_4bit: true + output_dir: outputs + num_epochs: 1 + learning_rate: 0.0002 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.01 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + +lora: + lora_r: 64 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: "q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj" + vision_all_linear: false + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: unsloth-training + enable_tensorboard: false + tensorboard_dir: runs diff --git a/studio/backend/assets/configs/model_defaults/default.yaml b/studio/backend/assets/configs/model_defaults/default.yaml new file mode 100644 index 0000000000..523775cff7 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/default.yaml @@ -0,0 +1,47 @@ +# Default model training parameters +# Used for models without specific configurations + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 5e-5 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_ratio: 0.1 + max_steps: 0 + save_steps: 0 + weight_decay: 0.01 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml new file mode 100644 index 0000000000..627d71b3fa --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-21B-A3B-PT.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/ERNIE-4.5-21B-A3B-PT +# Based on ERNIE_4_5_21B_A3B_PT-Conversational.ipynb +# Also applies to: unsloth/ERNIE-4.5-21B-A3B-PT + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 4 + gradient_accumulation_steps: 2 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml new file mode 100644 index 0000000000..04ae59d293 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/ernie/unsloth_ERNIE-4.5-VL-28B-A3B-PT.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/ERNIE-4.5-VL-28B-A3B-PT +# Based on ERNIE_4_5_VL_28B_A3B_PT_Vision.ipynb +# Also applies to: unsloth/ERNIE-4.5-VL-28B-A3B-PT + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 2 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml new file mode 100644 index 0000000000..2d3cfc1325 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/falcon/tiiuae_Falcon-H1-0.5B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for tiiuae/Falcon-H1-0.5B-Instruct +# Based on Falcon_H1_(0.5B)-Alpaca.ipynb +# Also applies to: tiiuae/Falcon-H1-0.5B-Instruct, unsloth/Falcon-H1-0.5B-Instruct + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 8 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.01 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: false + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.1 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml new file mode 100644 index 0000000000..02d0428f94 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_codegemma-7b-bnb-4bit.yaml @@ -0,0 +1,46 @@ +# Model defaults for unsloth/codegemma-7b-bnb-4bit +# Based on CodeGemma_(7B)-Conversational.ipynb +# Also applies to: unsloth/codegemma-7b, google/codegemma-7b + +training: + max_seq_length: 4096 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml new file mode 100644 index 0000000000..283ab50754 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_functiongemma-270m-it.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/functiongemma-270m-it +# Based on FunctionGemma_(270M).ipynb +# Also applies to: unsloth/functiongemma-270m-it-unsloth-bnb-4bit, google/functiongemma-270m-it, unsloth/functiongemma-270m-it-unsloth-bnb-4bit + +training: + max_seq_length: 4096 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 4 + gradient_accumulation_steps: 2 + warmup_steps: 10 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 128 + lora_alpha: 256 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml new file mode 100644 index 0000000000..899cdc01c7 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-27b-bnb-4bit.yaml @@ -0,0 +1,46 @@ +# Model defaults for unsloth/gemma-2-27b-bnb-4bit +# Based on Gemma2_(9B)-Alpaca.ipynb (same defaults for larger models) + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml new file mode 100644 index 0000000000..d56d64e9de --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-2-2b.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/gemma-2-2b +# Based on Gemma2_(2B)-Alpaca.ipynb +# Also applies to: unsloth/gemma-2-2b-bnb-4bit, google/gemma-2-2b + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.01 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml new file mode 100644 index 0000000000..eab0257381 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-270m-it.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/gemma-3-270m-it +# Based on Gemma3_(270M).ipynb +# Also applies to: unsloth/gemma-3-270m-it-unsloth-bnb-4bit, google/gemma-3-270m-it, unsloth/gemma-3-270m-it-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 5e-5 + batch_size: 4 + gradient_accumulation_steps: 1 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 128 + lora_alpha: 128 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml new file mode 100644 index 0000000000..c8fc2a8269 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-27b-it.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/gemma-3-27b-it +# Based on Gemma3_(27B)_A100-Conversational.ipynb +# Also applies to: unsloth/gemma-3-27b-it-unsloth-bnb-4bit, google/gemma-3-27b-it, unsloth/gemma-3-27b-it-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 8 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml new file mode 100644 index 0000000000..a9535537b4 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-it.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/gemma-3-4b-it +# Based on Gemma3_(4B).ipynb +# Also applies to: unsloth/gemma-3-4b-it-unsloth-bnb-4bit, google/gemma-3-4b-it, unsloth/gemma-3-4b-it-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 8 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml new file mode 100644 index 0000000000..80b85bb836 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3-4b-pt.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/gemma-3-4b-pt +# Based on Gemma3_(4B)-Vision.ipynb +# Also applies to: unsloth/gemma-3-4b-pt-unsloth-bnb-4bit, google/gemma-3-4b-pt, unsloth/gemma-3-4b-pt-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 2 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_ratio: 0.03 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: true + optim: "adamw_torch_fused" + lr_scheduler_type: "cosine" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml new file mode 100644 index 0000000000..1aeb4f59f8 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B-it.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/gemma-3n-E4B-it +# Based on Gemma3N_(4B)-Conversational.ipynb +# Also applies to: unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit, google/gemma-3n-E4B-it, unsloth/gemma-3n-E4B-it-unsloth-bnb-4bit + +training: + max_seq_length: 1024 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 8 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml new file mode 100644 index 0000000000..47cbc32782 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gemma/unsloth_gemma-3n-E4B.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/gemma-3n-E4B +# Based on Gemma3N_(4B)-Vision.ipynb +# Also applies to: unsloth/gemma-3n-E4B-unsloth-bnb-4bit, google/gemma-3n-E4B + +training: + max_seq_length: 2048 + num_epochs: 2 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_ratio: 0.03 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: true + optim: "adamw_torch_fused" + lr_scheduler_type: "cosine" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml new file mode 100644 index 0000000000..fc7dedc5fd --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-120b.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/gpt-oss-120b +# Based on gpt-oss-(120B)_A100-Fine-tuning.ipynb +# Also applies to: openai/gpt-oss-120b, unsloth/gpt-oss-120b-unsloth-bnb-4bit + +training: + max_seq_length: 4096 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 4 + gradient_accumulation_steps: 1 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml new file mode 100644 index 0000000000..d98dd28700 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/gpt-oss/unsloth_gpt-oss-20b.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/gpt-oss-20b +# Based on gpt-oss-(20B)-Fine-tuning.ipynb +# Also applies to: openai/gpt-oss-20b, unsloth/gpt-oss-20b-unsloth-bnb-4bit, unsloth/gpt-oss-20b-BF16 + +training: + max_seq_length: 1024 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml new file mode 100644 index 0000000000..718ec156f8 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-350m-unsloth-bnb-4bit.yaml @@ -0,0 +1,49 @@ +# Model defaults for unsloth/granite-4.0-350m +# Based on Granite4.0_350M.ipynb +# Also applies to: ibm-granite/granite-4.0-350m, unsloth/granite-4.0-350m-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + - "shared_mlp.input_linear" + - "shared_mlp.output_linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml new file mode 100644 index 0000000000..630366faee --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/granite/unsloth_granite-4.0-h-micro.yaml @@ -0,0 +1,49 @@ +# Model defaults for unsloth/granite-4.0-h-micro +# Based on Granite4.0.ipynb +# Also applies to: ibm-granite/granite-4.0-h-micro, unsloth/granite-4.0-h-micro-bnb-4bit, unsloth/granite-4.0-h-micro-unsloth-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + - "shared_mlp.input_linear" + - "shared_mlp.output_linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml new file mode 100644 index 0000000000..9147c93586 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-11B-Vision-Instruct.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/Llama-3.2-11B-Vision-Instruct +# Based on Llama3.2_(11B)-Vision.ipynb +# Also applies to: unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.2-11B-Vision-Instruct, unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml new file mode 100644 index 0000000000..1622d42bef --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-1B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Llama-3.2-1B-Instruct +# Based on Llama3.2_(1B)-RAFT.ipynb +# Also applies to: unsloth/Llama-3.2-1B-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.2-1B-Instruct, unsloth/Llama-3.2-1B-Instruct-bnb-4bit, RedHatAI/Llama-3.2-1B-Instruct-FP8, unsloth/Llama-3.2-1B-Instruct-FP8-Block, unsloth/Llama-3.2-1B-Instruct-FP8-Dynamic + +training: + max_seq_length: 2048 + num_epochs: 5 + learning_rate: 2e-5 + batch_size: 1 + gradient_accumulation_steps: 8 + warmup_steps: 0 + max_steps: 0 + save_steps: 0 + weight_decay: 0.01 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: true + optim: "adamw_torch" + lr_scheduler_type: "cosine" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml new file mode 100644 index 0000000000..217dbd50db --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.2-3B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Llama-3.2-3B-Instruct +# Based on Llama3.2_(1B_and_3B)-Conversational.ipynb +# Also applies to: unsloth/Llama-3.2-3B-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.2-3B-Instruct, unsloth/Llama-3.2-3B-Instruct-bnb-4bit, RedHatAI/Llama-3.2-3B-Instruct-FP8, unsloth/Llama-3.2-3B-Instruct-FP8-Block, unsloth/Llama-3.2-3B-Instruct-FP8-Dynamic + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml new file mode 100644 index 0000000000..166d43a788 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Llama-3.3-70B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Llama-3.3-70B-Instruct +# Based on Llama3.3_(70B)_A100-Conversational.ipynb +# Also applies to: unsloth/Llama-3.3-70B-Instruct-unsloth-bnb-4bit, meta-llama/Llama-3.3-70B-Instruct, unsloth/Llama-3.3-70B-Instruct-bnb-4bit, RedHatAI/Llama-3.3-70B-Instruct-FP8, unsloth/Llama-3.3-70B-Instruct-FP8-Block, unsloth/Llama-3.3-70B-Instruct-FP8-Dynamic + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml new file mode 100644 index 0000000000..411d7c2df8 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-70B-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Meta-Llama-3.1-70B-bnb-4bit +# Based on Llama3.1_(8B)-Alpaca.ipynb +# Also applies to: unsloth/Meta-Llama-3.1-8B-bnb-4bit, unsloth/Meta-Llama-3.1-8B-unsloth-bnb-4bit, meta-llama/Meta-Llama-3.1-8B, unsloth/Meta-Llama-3.1-8B, unsloth/Meta-Llama-3.1-70B, meta-llama/Meta-Llama-3.1-70B, unsloth/Meta-Llama-3.1-405B-bnb-4bit, meta-llama/Meta-Llama-3.1-405B + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml new file mode 100644 index 0000000000..5ba7d7b827 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_Meta-Llama-3.1-8B-Instruct-bnb-4bit.yaml @@ -0,0 +1,48 @@ +# Model defaults for unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit +# Based on Llama3.1_(8B)-Inference.ipynb +# Also applies to: "unsloth/Meta-Llama-3.1-8B-Instruct-unsloth-bnb-4bit", "meta-llama/Meta-Llama-3.1-8B-Instruct", "unsloth/Meta-Llama-3.1-8B-Instruct","RedHatAI/Llama-3.1-8B-Instruct-FP8","unsloth/Llama-3.1-8B-Instruct-FP8-Block","unsloth/Llama-3.1-8B-Instruct-FP8-Dynamic" + +training: + max_seq_length: 8192 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml new file mode 100644 index 0000000000..aa5aeb178b --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-Instruct-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/llama-3-8b-Instruct-bnb-4bit +# Based on Llama3_(8B)-Conversational.ipynb +# Also applies to: unsloth/llama-3-8b-Instruct, meta-llama/Meta-Llama-3-8B-Instruct + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml new file mode 100644 index 0000000000..be515b0bc3 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llama/unsloth_llama-3-8b-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/llama-3-8b-bnb-4bit +# Based on Llama3_(8B)-Alpaca.ipynb +# Also applies to: unsloth/llama-3-8b, meta-llama/Meta-Llama-3-8B + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml b/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml new file mode 100644 index 0000000000..79e5c9d692 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/llasa/unsloth_Llasa-3B.yaml @@ -0,0 +1,42 @@ +# Model defaults for unsloth/Llasa-3B +# Based on Llasa_TTS_(3B).ipynb and Llasa_TTS_(1B).ipynb +# Also applies to: HKUSTAudio/Llasa-1B + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 5e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 128 + lora_alpha: 128 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "v_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml new file mode 100644 index 0000000000..bb605451a5 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Magistral-Small-2509-unsloth-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Magistral-Small-2509 +# Based on Magistral_(24B)-Reasoning-Conversational.ipynb +# Also applies to: mistralai/Magistral-Small-2509, unsloth/Magistral-Small-2509-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 2 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml new file mode 100644 index 0000000000..b5fd0f5713 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Ministral-3-3B-Instruct-2512.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Ministral-3-3B-Instruct-2512 +# Based on Ministral_3_VL_(3B)_Vision.ipynb +# Also applies to: unsloth/Ministral-3-3B-Instruct-2512 + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 4 + gradient_accumulation_steps: 2 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml new file mode 100644 index 0000000000..d1374cc49e --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Nemo-Base-2407-bnb-4bit.yaml @@ -0,0 +1,48 @@ +# Model defaults for unsloth/Mistral-Nemo-Base-2407-bnb-4bit +# Based on Mistral_Nemo_(12B)-Alpaca.ipynb +# Also applies to: "unsloth/Mistral-Nemo-Base-2407", "mistralai/Mistral-Nemo-Base-2407", "unsloth/Mistral-Nemo-Instruct-2407-bnb-4bit", "unsloth/Mistral-Nemo-Instruct-2407", "mistralai/Mistral-Nemo-Instruct-2407", + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml new file mode 100644 index 0000000000..597cf612fe --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Mistral-Small-Instruct-2409.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Mistral-Small-Instruct-2409 +# Based on Mistral_Small_(22B)-Alpaca.ipynb +# Also applies to: unsloth/Mistral-Small-Instruct-2409-bnb-4bit, mistralai/Mistral-Small-Instruct-2409 + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml new file mode 100644 index 0000000000..7945f1944c --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_Pixtral-12B-2409.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/Pixtral-12B-2409 +# Based on Pixtral_(12B)-Vision.ipynb +# Also applies to: unsloth/Pixtral-12B-2409-unsloth-bnb-4bit, mistralai/Pixtral-12B-2409, unsloth/Pixtral-12B-2409-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "paged_adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 8 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: false + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml new file mode 100644 index 0000000000..384de103d0 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-instruct-v0.3-bnb-4bit.yaml @@ -0,0 +1,48 @@ +# Model defaults for unsloth/mistral-7b-instruct-v0.3-bnb-4bit +# Based on Mistral_v0.3_(7B)-Conversational.ipynb +# Also applies to: unsloth/mistral-7b-instruct-v0.3, mistralai/Mistral-7B-Instruct-v0.3 + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + + diff --git a/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml new file mode 100644 index 0000000000..fe288786aa --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/mistral/unsloth_mistral-7b-v0.3-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/mistral-7b-v0.3-bnb-4bit +# Based on Mistral_v0.3_(7B)-Alpaca.ipynb +# Also applies to: "unsloth/mistral-7b-v0.3", "mistralai/Mistral-7B-v0.3", +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + + diff --git a/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml b/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml new file mode 100644 index 0000000000..67cc14b38d --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/OuteAI_Llama-OuteTTS-1.0-1B.yaml @@ -0,0 +1,42 @@ +# Model defaults for OuteAI/Llama-OuteTTS-1.0-1B +# Based on Oute_TTS_(1B).ipynb +# Also applies to: OuteAI/Llama-OuteTTS-1.0-1B + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 128 + lora_alpha: 128 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "v_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml b/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml new file mode 100644 index 0000000000..0c08015a7f --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/Spark-TTS-0.5B_LLM.yaml @@ -0,0 +1,47 @@ +# Model defaults for Spark-TTS-0.5B/LLM +# Based on Spark_TTS_(0_5B).ipynb +# Also applies to: Spark-TTS-0.5B/LLM + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 128 + lora_alpha: 128 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml b/studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml new file mode 100644 index 0000000000..a01e10971b --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/sesame_csm-1b.yaml @@ -0,0 +1,47 @@ +# Model defaults for sesame/csm-1b +# Based on Sesame_CSM_(1B)-TTS.ipynb +# Also applies to: sesame/csm-1b + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml new file mode 100644 index 0000000000..f51d88b427 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_LFM2-1.2B.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/LFM2-1.2B +# Based on Liquid_LFM2_(1.2B)-Conversational.ipynb +# Also applies to: unsloth/LFM2-1.2B + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml new file mode 100644 index 0000000000..f40a2fd620 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_Nemotron-3-Nano-30B-A3B.yaml @@ -0,0 +1,49 @@ +# Model defaults for unsloth/Nemotron-3-Nano-30B-A3B +# Based on Nemotron-3-Nano-30B-A3B_A100.ipynb +# Also applies to: unsloth/Nemotron-3-Nano-30B-A3B + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 4 + gradient_accumulation_steps: 2 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + - "in_proj" + - "out_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml new file mode 100644 index 0000000000..370b17c0a3 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_PaddleOCR-VL.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/PaddleOCR-VL +# Based on Paddle_OCR_(1B)_Vision.ipynb +# Also applies to: unsloth/PaddleOCR-VL + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 5e-5 + batch_size: 4 + gradient_accumulation_steps: 2 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 64 + lora_alpha: 64 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml new file mode 100644 index 0000000000..3b886fb3b9 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_answerdotai_ModernBERT-large.yaml @@ -0,0 +1,46 @@ +# Model defaults for answerdotai/ModernBERT-large +# Based on bert_classification.ipynb + +training: + max_seq_length: 2048 + num_epochs: 1 + learning_rate: 5e-5 + batch_size: 32 + gradient_accumulation_steps: 1 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml new file mode 100644 index 0000000000..3eac4fdde5 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_orpheus-3b-0.1-ft.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/orpheus-3b-0.1-ft +# Based on Orpheus_(3B)-TTS.ipynb +# Also applies to: unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit, canopylabs/orpheus-3b-0.1-ft, unsloth/orpheus-3b-0.1-ft-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 64 + lora_alpha: 64 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml new file mode 100644 index 0000000000..f5f7ae79c3 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_tinyllama-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/tinyllama +# Based on TinyLlama_(1.1B)-Alpaca.ipynb +# Also applies to: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T + +training: + max_seq_length: 4096 + num_epochs: 1 + learning_rate: 2e-5 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_ratio: 0.1 + max_steps: 0 + save_steps: 0 + weight_decay: 0.1 + random_seed: 3407 + packing: true + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml new file mode 100644 index 0000000000..3b1c82ab0a --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_whisper-large-v3.yaml @@ -0,0 +1,42 @@ +# Model defaults for unsloth/whisper-large-v3 +# Based on Whisper.ipynb +# Also applies to: unsloth/whisper-large-v3, openai/whisper-large-v3 + +training: + max_seq_length: 448 + num_epochs: 4 + learning_rate: 1e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 64 + lora_alpha: 64 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "v_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml new file mode 100644 index 0000000000..5968e9aea8 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3-medium-4k-instruct.yaml @@ -0,0 +1,48 @@ +# Model defaults for unsloth/Phi-3-medium-4k-instruct +# Based on Phi_3_Medium-Conversational.ipynb +# Also applies to: "unsloth/Phi-3-medium-4k-instruct-bnb-4bit", "microsoft/Phi-3-medium-4k-instruct", + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + + diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml new file mode 100644 index 0000000000..8a2e2ca26e --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-3.5-mini-instruct.yaml @@ -0,0 +1,48 @@ +# Model defaults for unsloth/Phi-3.5-mini-instruct +# Based on Phi_3.5_Mini-Conversational.ipynb +# Also applies to: "unsloth/Phi-3.5-mini-instruct-bnb-4bit", "microsoft/Phi-3.5-mini-instruct" + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + + diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4-14B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4-14B-Instruct.yaml new file mode 100644 index 0000000000..6b41ca81f2 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4-14B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Phi-4-14B-Instruct +# Based on Phi_4-Conversational.ipynb +# Also applies to: unsloth/Phi-4-14B-Instruct-unsloth-bnb-4bit, microsoft/Phi-4-14B-Instruct, unsloth/Phi-4-14B-Instruct-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml new file mode 100644 index 0000000000..9746e01374 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/phi/unsloth_Phi-4.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Phi-4 +# Based on Phi_4-Conversational.ipynb +# Also applies to: unsloth/phi-4-unsloth-bnb-4bit, microsoft/phi-4, unsloth/phi-4-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml new file mode 100644 index 0000000000..69dbd27626 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-7B.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen2-7B +# Based on Qwen2_(7B)-Alpaca.ipynb +# Also applies to: unsloth/Qwen2-7B-bnb-4bit, Qwen/Qwen2-7B + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml new file mode 100644 index 0000000000..7394967628 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2-VL-7B-Instruct.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/Qwen2-VL-7B-Instruct +# Based on Qwen2_VL_(7B)-Vision.ipynb +# Also applies to: unsloth/Qwen2-VL-7B-Instruct-unsloth-bnb-4bit, Qwen/Qwen2-VL-7B-Instruct, unsloth/Qwen2-VL-7B-Instruct-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml new file mode 100644 index 0000000000..1f08adf6d2 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-1.5B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen2.5-1.5B-Instruct +# Based on nemo_gym_sudoku.ipynb +# Also applies to: unsloth/Qwen2.5-1.5B-Instruct-unsloth-bnb-4bit, Qwen/Qwen2.5-1.5B-Instruct, unsloth/Qwen2.5-1.5B-Instruct-bnb-4bit + +training: + max_seq_length: 4096 + num_epochs: 4 + learning_rate: 1e-5 + batch_size: 1 + gradient_accumulation_steps: 64 + warmup_ratio: 0.1 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 42 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 4 + lora_alpha: 8 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml new file mode 100644 index 0000000000..31ab069907 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-7B.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen2.5-7B +# Based on Qwen2.5_(7B)-Alpaca.ipynb +# Also applies to: unsloth/Qwen2.5-7B-unsloth-bnb-4bit, Qwen/Qwen2.5-7B, unsloth/Qwen2.5-7B-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml new file mode 100644 index 0000000000..e340ea4573 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-1.5B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen2.5-Coder-1.5B-Instruct +# Based on Qwen2.5_Coder_(1.5B)-Tool_Calling.ipynb +# Also applies to: unsloth/Qwen2.5-Coder-1.5B-Instruct-bnb-4bit, Qwen/Qwen2.5-Coder-1.5B-Instruct + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml new file mode 100644 index 0000000000..3b45853b6c --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-14B-Instruct.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen2.5-Coder-14B-Instruct +# Based on Qwen2.5_Coder_(14B)-Conversational.ipynb +# Also applies to: unsloth/Qwen2.5-Coder-14B-Instruct-bnb-4bit, Qwen/Qwen2.5-Coder-14B-Instruct + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "paged_adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml new file mode 100644 index 0000000000..ae62611432 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-Coder-7B-Instruct-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen2.5-Coder-7B-Instruct-bnb-4bit +# Based on CodeForces-cot-Finetune_for_Reasoning_on_CodeForces.ipynb +# Also applies to: unsloth/Qwen2.5-Coder-7B-Instruct, Qwen/Qwen2.5-Coder-7B-Instruct + +training: + max_seq_length: 32768 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml new file mode 100644 index 0000000000..daa92cf5ed --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen2.5-VL-7B-Instruct-bnb-4bit.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit +# Based on Qwen2.5_VL_(7B)-Vision.ipynb +# Also applies to: unsloth/Qwen2.5-VL-7B-Instruct, Qwen/Qwen2.5-VL-7B-Instruct, unsloth/Qwen2.5-VL-7B-Instruct-unsloth-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml new file mode 100644 index 0000000000..153e0d13e1 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-0.6B.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen3-0.6B +# Based on Qwen3_(0_6B)-Phone_Deployment.ipynb +# Also applies to: unsloth/Qwen3-0.6B-unsloth-bnb-4bit, Qwen/Qwen3-0.6B, unsloth/Qwen3-0.6B-bnb-4bit, Qwen/Qwen3-0.6B-FP8, unsloth/Qwen3-0.6B-FP8 + +training: + max_seq_length: 1024 + num_epochs: 4 + learning_rate: 5e-5 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml new file mode 100644 index 0000000000..76dd3de761 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B-Base-unsloth-bnb-4bit.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen3-14B-Base +# Based on Qwen3_(14B)-Alpaca.ipynb +# Also applies to: unsloth/Qwen3-14B-Base, Qwen/Qwen3-14B-Base, unsloth/Qwen3-14B-Base-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r:16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml new file mode 100644 index 0000000000..51bcfc5501 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-14B.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen3-14B +# Based on Qwen3_(14B).ipynb +# Also applies to: unsloth/Qwen3-14B-unsloth-bnb-4bit, Qwen/Qwen3-14B, unsloth/Qwen3-14B-bnb-4bit, Qwen/Qwen3-14B-FP8, unsloth/Qwen3-14B-FP8 + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml new file mode 100644 index 0000000000..76098e80f5 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-32B.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen3-32B +# Based on Qwen3_(32B)_A100-Reasoning-Conversational.ipynb +# Also applies to: unsloth/Qwen3-32B-unsloth-bnb-4bit, Qwen/Qwen3-32B, unsloth/Qwen3-32B-bnb-4bit, Qwen/Qwen3-32B-FP8, unsloth/Qwen3-32B-FP8 + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml new file mode 100644 index 0000000000..1a1efaf975 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Instruct-2507.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen3-4B-Instruct-2507 +# Based on Qwen3_(4B)-Instruct.ipynb +# Also applies to: unsloth/Qwen3-4B-Instruct-2507-unsloth-bnb-4bit, Qwen/Qwen3-4B-Instruct-2507, unsloth/Qwen3-4B-Instruct-2507-bnb-4bit, Qwen/Qwen3-4B-Instruct-2507-FP8, unsloth/Qwen3-4B-Instruct-2507-FP8 + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml new file mode 100644 index 0000000000..5a67224009 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-4B-Thinking-2507.yaml @@ -0,0 +1,47 @@ +# Model defaults for unsloth/Qwen3-4B-Thinking-2507 +# Based on Qwen3_(4B)-Thinking.ipynb +# Also applies to: unsloth/Qwen3-4B-Thinking-2507-unsloth-bnb-4bit, Qwen/Qwen3-4B-Thinking-2507, unsloth/Qwen3-4B-Thinking-2507-bnb-4bit, Qwen/Qwen3-4B-Thinking-2507-FP8, unsloth/Qwen3-4B-Thinking-2507-FP8 + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 32 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + use_rslora: false + use_loftq: false + finetune_vision_layers: false + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml new file mode 100644 index 0000000000..56b2feb653 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-VL-8B-Instruct-unsloth-bnb-4bit.yaml @@ -0,0 +1,41 @@ +# Model defaults for unsloth/Qwen3-VL-8B-Instruct +# Based on Qwen3_VL_(8B)-Vision.ipynb +# Also applies to: Qwen/Qwen3-VL-8B-Instruct-FP8, unsloth/Qwen3-VL-8B-Instruct-FP8, unsloth/Qwen3-VL-8B-Instruct, Qwen/Qwen3-VL-8B-Instruct, unsloth/Qwen3-VL-8B-Instruct-bnb-4bit + +training: + max_seq_length: 2048 + num_epochs: 4 + learning_rate: 2e-4 + batch_size: 2 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 16 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "all-linear" + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + diff --git a/studio/backend/assets/configs/vision_lora.yaml b/studio/backend/assets/configs/vision_lora.yaml new file mode 100644 index 0000000000..60641b16e8 --- /dev/null +++ b/studio/backend/assets/configs/vision_lora.yaml @@ -0,0 +1,42 @@ +model: unsloth/Qwen2-VL-2B-Instruct-bnb-4bit + +data: + dataset: philschmid/amazon-product-descriptions-vlm + format_type: auto + +training: + training_type: lora + max_seq_length: 2048 + load_in_4bit: true + output_dir: outputs + num_epochs: 1 + learning_rate: 0.0002 + batch_size: 1 + gradient_accumulation_steps: 4 + warmup_steps: 5 + max_steps: 0 + save_steps: 0 + weight_decay: 0.01 + random_seed: 3407 + packing: false + train_on_completions: false + gradient_checkpointing: "unsloth" + +lora: + lora_r: 64 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: "" # vision uses vision_all_linear by default + vision_all_linear: true + use_rslora: false + use_loftq: false + finetune_vision_layers: true + finetune_language_layers: true + finetune_attention_modules: true + finetune_mlp_modules: true + +logging: + enable_wandb: false + wandb_project: unsloth-training + enable_tensorboard: false + tensorboard_dir: runs