ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update cross_entropy_loss.py

Browse source
This commit is contained in:
Daniel Han 2025-03-02 23:08:44 -08:00
commit 72a7919e37
1 changed files with 4 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

7
unsloth/kernels/cross_entropy_loss.py
View file

@ -279,10 +279,11 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
n_rows : int
vocab_size : int
n_rows, vocab_size = logits.shape
device = logits.device
div, mod = divmod(vocab_size, MAX_FUSED_SIZE)
n_chunks : int = div + (mod != 0)
losses = torch.empty(n_rows, dtype = torch.float32, device = "cuda:0")
losses = torch.empty(n_rows, dtype = torch.float32, device = device)
DO_SOFTCAPPING : bool = bool(logit_softcapping != 0)
DO_LOGIT_SCALING : bool = bool(logit_scaling != 0)
@ -292,7 +293,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
if n_chunks == 1:
# For small vocabs <= 65336 like Llama, Mistral
BLOCK_SIZE, num_warps = calculate_settings(vocab_size)
logsumexp = torch.empty(n_rows, dtype = torch.float32, device = "cuda:0")
logsumexp = torch.empty(n_rows, dtype = torch.float32, device = device)
_cross_entropy_forward[(n_rows,)](
logits, logits.stride(0),
@ -309,7 +310,7 @@ class Fast_CrossEntropyLoss(torch.autograd.Function):
)
else:
# For large vocabs > 65336 like Gemma 256K
logsumexp = torch.empty((n_rows, n_chunks,), dtype = torch.float32, device = "cuda:0")
logsumexp = torch.empty((n_rows, n_chunks,), dtype = torch.float32, device = device)
_chunked_cross_entropy_forward[(n_rows, n_chunks,)](
logits, logits.stride(0),