ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: add support for init_lora_weights="corda" in get_peft_model (#3794)

Browse source 
Add "corda" as an allowed value for the init_lora_weights parameter
in FastLanguageModel.get_peft_model() and FastBaseModel.get_peft_model().

This enables users to use CorDA (Correlation-aware Decomposed Adaptation)
initialization from PEFT, which provides an alternative LoRA initialization
strategy for improved finetuning performance.

Fixes #3693

Signed-off-by: majiayu000 <1835304752@qq.com>
This commit is contained in:
lif 2025-12-29 15:17:58 +08:00 • committed by GitHub
commit 9fedb1c11d
2 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
unsloth/models/_utils.py
View file

@ -1981,9 +1981,10 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m
type(init_lora_weights) is bool
or init_lora_weights == "gaussian"
or init_lora_weights == "loftq"
or init_lora_weights == "corda"
):
raise ValueError(
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].'
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
)
if init_lora_weights == "loftq":

3
unsloth/models/llama.py
View file

@ -2779,9 +2779,10 @@ class FastLlamaModel:
type(init_lora_weights) is bool
or init_lora_weights == "gaussian"
or init_lora_weights == "loftq"
or init_lora_weights == "corda"
):
raise ValueError(
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].'
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
)
if init_lora_weights == "loftq":