From b30d2502ccb8f97e5c0ec252a36ec6dc04f20f4b Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Fri, 23 Feb 2024 02:12:02 +1100 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 32d159510c..444ae75549 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1317,7 +1317,7 @@ class FastLlamaModel: pass # Get activation function - model_type = model_config.model_type + model_type = model.config.model_type if model_type == "llama": apply_lora_mlp = apply_lora_mlp_swiglu elif model_type == "mistral": apply_lora_mlp = apply_lora_mlp_swiglu @@ -1325,7 +1325,7 @@ class FastLlamaModel: else: raise NotImplementedError(f"Unsloth: {model_type} is not yet implemented!") pass - + model = prepare_model_for_kbit_training( model, use_gradient_checkpointing = use_gradient_checkpointing,