From 9bf61e65d97dda9bc8d9b91ddeb769cf6f018d6f Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 30 Oct 2024 16:46:33 -0700 Subject: [PATCH] Update cross_entropy_loss.py --- unsloth/kernels/cross_entropy_loss.py | 1 + 1 file changed, 1 insertion(+) diff --git a/unsloth/kernels/cross_entropy_loss.py b/unsloth/kernels/cross_entropy_loss.py index c5f054a43f..474c291ed8 100644 --- a/unsloth/kernels/cross_entropy_loss.py +++ b/unsloth/kernels/cross_entropy_loss.py @@ -232,6 +232,7 @@ def _cross_entropy_backward( pass # Do logit softcapping for Gemma 2: t * tanh(1/t * x) + partial = 0.0 if DO_SOFTCAPPING: # d/dx [t * tanh(1/t * x)] = 1 - tanh^2(1/t * x) partial = triton_tanh(x / SOFTCAP)