From e699f1d09e38817cc10621031d7e92edc799a456 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Tue, 12 Dec 2023 00:57:06 +1100 Subject: [PATCH] Update llama.py --- unsloth/models/llama.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 1dc69be042..9e9c3cca5d 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -411,7 +411,7 @@ def LlamaModel_fast_forward( (batch_size, seq_length), inputs_embeds, past_key_values_length, - sliding_window = None if not hasattr(self.config "sliding_window") else \ + sliding_window = None if not hasattr(self.config, "sliding_window") else \ self.config.sliding_window, ) pass @@ -821,7 +821,7 @@ class FastLlamaModel: layer.self_attn.apply_o = apply_lora_o pass pass - + # Patch cross entropy loss labels # Fixes https://github.com/unslothai/unsloth/issues/10 extra_ignored_labels = torch.full((max_seq_length, 1), -100, device = "cuda")