From 3edc9cf1785bdde65e5a079a57ca61a48da0215a Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 12 Mar 2026 10:01:32 +0000 Subject: [PATCH] Add target_parameters and use_dora support for MoE models --- unsloth/models/llama.py | 42 ++++++++++++++++++++++++++++++++++++++++ unsloth/models/vision.py | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 93d93e26d6..0e025d6441 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2744,6 +2744,7 @@ class FastLlamaModel: random_state = 3407, max_seq_length = 2048, # not used anymore use_rslora = False, + use_dora = False, modules_to_save = None, init_lora_weights = True, loftq_config = {}, @@ -2776,6 +2777,7 @@ class FastLlamaModel: random_state = random_state, max_seq_length = max_seq_length, use_rslora = use_rslora, + use_dora = use_dora, modules_to_save = modules_to_save, init_lora_weights = init_lora_weights, loftq_config = loftq_config, @@ -2887,6 +2889,7 @@ class FastLlamaModel: signature = str(inspect.signature(LoraConfig)) SUPPORTS_LOFTQ = "loftq_config" in signature SUPPORTS_RSLORA = "use_rslora" in signature + SUPPORTS_DORA = "use_dora" in signature if lora_dropout != 0: logger.warning_once( @@ -2900,6 +2903,35 @@ class FastLlamaModel: f"Unsloth will patch all other layers, except LoRA matrices, causing a performance hit." ) + if target_parameters is not None: + if lora_dropout != 0: + raise ValueError( + "Unsloth: target_parameters does not support lora_dropout != 0.\n" + "Please set lora_dropout = 0 when using target_parameters." + ) + if use_dora: + raise ValueError( + "Unsloth: target_parameters does not support use_dora = True.\n" + "Please set use_dora = False when using target_parameters." + ) + if kwargs.get("lora_bias", False): + raise ValueError( + "Unsloth: target_parameters does not support lora_bias = True.\n" + "Please set lora_bias = False when using target_parameters." + ) + invalid_params = [ + p + for p in target_parameters + if p + in ("lm_head.weight", "lm_head", "embed_tokens.weight", "embed_tokens") + ] + if invalid_params: + raise ValueError( + f"Unsloth: target_parameters should not contain {invalid_params}.\n" + f"target_parameters is for targeting nn.Parameter objects directly (e.g., MoE expert weights).\n" + f"For embed_tokens/lm_head, use modules_to_save instead for full fine-tuning." + ) + if not ( type(init_lora_weights) is bool or init_lora_weights == "gaussian" @@ -2946,6 +2978,14 @@ class FastLlamaModel: "Please install PEFT 0.7.2 or higher.\n" "You can also install from source: `pip install git+https://github.com/huggingface/peft.git" ) + assert type(use_dora) is bool + if use_dora and not SUPPORTS_DORA: + import peft + + raise RuntimeError( + f"Unsloth: Your PEFT version of {peft.__version__} does not support `use_dora`.\n" + "Please install a newer PEFT version or disable use_dora." + ) accepted_modules = frozenset( ( @@ -3089,6 +3129,8 @@ class FastLlamaModel: del arguments["loftq_config"] if not SUPPORTS_RSLORA: del arguments["use_rslora"] + else: + arguments["use_dora"] = use_dora _saved_temp_tokenizer = model._saved_temp_tokenizer diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index a8adba99e7..b817338f48 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -1171,6 +1171,7 @@ class FastBaseModel: random_state = 3407, max_seq_length = 2048, # not used anymore use_rslora = False, + use_dora = False, modules_to_save = None, init_lora_weights = True, loftq_config = {}, @@ -1259,6 +1260,36 @@ class FastBaseModel: loftq_config, lora_dropout, bias, init_lora_weights, model ) + if target_parameters is not None: + if lora_dropout != 0: + raise ValueError( + "Unsloth: target_parameters does not support lora_dropout != 0.\n" + "Please set lora_dropout = 0 when using target_parameters." + ) + if use_dora: + raise ValueError( + "Unsloth: target_parameters does not support use_dora = True.\n" + "Please set use_dora = False when using target_parameters." + ) + if kwargs.get("lora_bias", False): + raise ValueError( + "Unsloth: target_parameters does not support lora_bias = True.\n" + "Please set lora_bias = False when using target_parameters." + ) + + invalid_params = [ + p + for p in target_parameters + if p + in ("lm_head.weight", "lm_head", "embed_tokens.weight", "embed_tokens") + ] + if invalid_params: + raise ValueError( + f"Unsloth: target_parameters should not contain {invalid_params}.\n" + f"target_parameters is for targeting nn.Parameter objects directly (e.g., MoE expert weights).\n" + f"For embed_tokens/lm_head, use modules_to_save instead for full fine-tuning." + ) + # Auto-detect MoE models and populate target_parameters for expert layers if target_parameters is None: target_parameters = get_moe_target_parameters(model, target_modules) @@ -1270,6 +1301,13 @@ class FastBaseModel: } del local_variables["kwargs"] allowed_parameters = inspect.signature(LoraConfig).parameters.keys() + if use_dora and "use_dora" not in allowed_parameters: + import peft + + raise RuntimeError( + f"Unsloth: Your PEFT version of {peft.__version__} does not support `use_dora`.\n" + "Please install a newer PEFT version or disable use_dora." + ) lora_config = LoraConfig( **{k: v for k, v in local_variables.items() if k in allowed_parameters}, )