ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update llama.py

Browse source
This commit is contained in:
Daniel Han-Chen 2023-12-12 00:57:06 +11:00
commit e699f1d09e
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
unsloth/models/llama.py
View file

@ -411,7 +411,7 @@ def LlamaModel_fast_forward(
(batch_size, seq_length),
inputs_embeds,
past_key_values_length,
sliding_window = None if not hasattr(self.config "sliding_window") else \
sliding_window = None if not hasattr(self.config, "sliding_window") else \
self.config.sliding_window,
)
pass
@ -821,7 +821,7 @@ class FastLlamaModel:
layer.self_attn.apply_o = apply_lora_o
pass
pass
# Patch cross entropy loss labels
# Fixes https://github.com/unslothai/unsloth/issues/10
extra_ignored_labels = torch.full((max_seq_length, 1), -100, device = "cuda")