From fb1c321ad330ccac6565551146aa8564c71e9efc Mon Sep 17 00:00:00 2001 From: Roland Tannous Date: Mon, 23 Feb 2026 05:51:43 +0000 Subject: [PATCH] Add GLM, Qwen3 MoE, TinyQwen3 MoE, and Ministral 3 VL model defaults and GLM train_on_responses_only mapping --- .../other/unsloth_GLM-4.7-Flash.yaml | 50 ++++++++++++++++++ .../imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml | 51 +++++++++++++++++++ .../unsloth_Qwen3-30B-A3B-Instruct-2507.yaml | 51 +++++++++++++++++++ .../backend/utils/datasets/model_mappings.py | 13 ++++- 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml create mode 100644 studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml create mode 100644 studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml diff --git a/studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml b/studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml new file mode 100644 index 0000000000..a973c2d4e4 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/other/unsloth_GLM-4.7-Flash.yaml @@ -0,0 +1,50 @@ +# Model defaults for unsloth/GLM-4.7-Flash +# Based on GLM_Flash_A100(80GB).py +# Also applies to: unsloth/GLM-4.7-Flash-unsloth-bnb-4bit, unsloth/GLM-4.7-Flash-bnb-4bit, THUDM/GLM-4.7-Flash + +training: + max_seq_length: 2048 + # num_epochs: 4 + num_epochs: 0 + learning_rate: 2e-4 + batch_size: 4 + gradient_accumulation_steps: 2 + warmup_steps: 5 + max_steps: 60 + save_steps: 60 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 8 + lora_alpha: 16 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + - "out_proj" + use_rslora: false + use_loftq: false + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + +inference: + temperature: 0.7 + top_p: 0.8 + top_k: 20 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml b/studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml new file mode 100644 index 0000000000..bb75b3ce52 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/imdatta0_tiny_qwen3_moe_2.8B_0.7B.yaml @@ -0,0 +1,51 @@ +# Model defaults for imdatta0/tiny_qwen3_moe_2.8B_0.7B +# Based on TinyQwen3_MoE.py +# Dummy model of qwen3moe architecture created to fit in T4 +# MoE model - includes gate_up_proj for MoE layers + +training: + max_seq_length: 2048 + # num_epochs: 4 + num_epochs: 0 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 1 + warmup_steps: 5 + max_steps: 50 + save_steps: 50 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 64 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + - "gate_up_proj" + use_rslora: false + use_loftq: false + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + +inference: + temperature: 0.6 + top_k: 20 + top_p: 0.95 + diff --git a/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml new file mode 100644 index 0000000000..2fb3a95c30 --- /dev/null +++ b/studio/backend/assets/configs/model_defaults/qwen/unsloth_Qwen3-30B-A3B-Instruct-2507.yaml @@ -0,0 +1,51 @@ +# Model defaults for unsloth/Qwen3-30B-A3B-Instruct-2507 +# Based on Qwen3_MoE.py +# Also applies to: Qwen/Qwen3-30B-A3B-Instruct-2507, unsloth/Qwen3-30B-A3B-Instruct-2507-bnb-4bit +# MoE model - includes gate_up_proj for MoE layers + +training: + max_seq_length: 2048 + # num_epochs: 4 + num_epochs: 0 + learning_rate: 2e-4 + batch_size: 1 + gradient_accumulation_steps: 1 + warmup_steps: 5 + max_steps: 50 + save_steps: 50 + weight_decay: 0.001 + random_seed: 3407 + packing: false + train_on_completions: true + gradient_checkpointing: "unsloth" + optim: "adamw_8bit" + lr_scheduler_type: "linear" + +lora: + lora_r: 32 + lora_alpha: 64 + lora_dropout: 0.0 + target_modules: + - "q_proj" + - "k_proj" + - "v_proj" + - "o_proj" + - "gate_proj" + - "up_proj" + - "down_proj" + - "gate_up_proj" + use_rslora: false + use_loftq: false + +logging: + enable_wandb: false + wandb_project: "llm-finetuning" + enable_tensorboard: false + tensorboard_dir: "runs" + log_frequency: 10 + +inference: + temperature: 0.6 + top_k: 20 + top_p: 0.95 + diff --git a/studio/backend/utils/datasets/model_mappings.py b/studio/backend/utils/datasets/model_mappings.py index 0691a15b29..22b0f4334b 100644 --- a/studio/backend/utils/datasets/model_mappings.py +++ b/studio/backend/utils/datasets/model_mappings.py @@ -383,7 +383,13 @@ TEMPLATE_TO_MODEL_MAPPER = { "unsloth/yi-34b-chat-bnb-4bit", "01-ai/Yi-6B-Chat", "01-ai/Yi-34B-Chat", - ) + ), + "glm": ( + "unsloth/GLM-4.7-Flash-unsloth-bnb-4bit", + "unsloth/GLM-4.7-Flash", + "THUDM/GLM-4.7-Flash", + "unsloth/GLM-4.7-Flash-bnb-4bit", + ), } MODEL_TO_TEMPLATE_MAPPER = {} @@ -506,4 +512,9 @@ TEMPLATE_TO_RESPONSES_MAPPER = { "instruction": "<|im_start|>user\n", "response": "<|im_start|>assistant\n", }, + "glm": { + "instruction": "[gMASK]<|user|>", + "response": "<|assistant|>", + }, } +