ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update layernorm.py

Browse source
This commit is contained in:
Daniel Han 2024-09-24 17:00:23 -07:00
commit 6801979ef3
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
unsloth/kernels/layernorm.py
View file

@ -178,7 +178,7 @@ def test_layernorm(
YY = torch.randn((bsz, seqlen, dim), dtype = dtype, device = "cuda", requires_grad = True)
Y.backward(YY)
correct_grad = X.grad.clone()
from unsloth.kernels import fast_layernorm
# from unsloth.kernels import fast_layernorm
Y = fast_layernorm(layernorm, XX)
Y.backward(YY)
assert(torch.dist(correct_grad, XX.grad).item() <= 0.1)