From 9a2e791e6c831137f5017d4bf7fa1741a3b06b0e Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sun, 3 Mar 2024 03:35:21 +1100 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 87ca5d8487..20552644d9 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -511,7 +511,12 @@ def LlamaModel_fast_forward( # Mormalized from Gemma if self.config.model_type == "gemma": inputs_requires_grad = inputs_embeds.requires_grad - if inputs_requires_grad: inputs_embeds.requires_grad_(False) + if not inputs_embeds.is_leaf: + inputs_embeds = inputs_embeds.detach() + inputs_requires_grad = True + elif inputs_requires_grad: + inputs_embeds.requires_grad_(False) + pass inputs_embeds *= math_sqrt(self.config.hidden_size) if inputs_requires_grad: inputs_embeds.requires_grad_(True) pass @@ -522,7 +527,12 @@ def LlamaModel_fast_forward( # Careful for inference the attention_mask is size (1, kv_seq_len) # Whilst the input_embeds is size (1, 1, 4096) inputs_requires_grad = inputs_embeds.requires_grad - if inputs_requires_grad: inputs_embeds.requires_grad_(False) + if not inputs_embeds.is_leaf: + inputs_embeds = inputs_embeds.detach() + inputs_requires_grad = True + elif inputs_requires_grad: + inputs_embeds.requires_grad_(False) + pass inputs_embeds *= attention_mask.unsqueeze(0).transpose(0, 1).transpose(1, 2) if inputs_requires_grad: inputs_embeds.requires_grad_(True) pass