ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update layernorm.py

Browse source
This commit is contained in:
Daniel Han 2024-09-24 17:03:19 -07:00
commit 80b440f5e2
1 changed files with 12 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

22
unsloth/kernels/layernorm.py
View file

@ -188,16 +188,18 @@ pass
def testing_suite_layernorm():
for dim in [512, 1024, 2048]:
for dtype in [torch.float16, torch.bfloat16]:
for seqlen in [3341, 2048, 349]:
for random_state in [3407, 42]:
test_layernorm(
dim = dim,
eps = 1e-5,
dtype = dtype,
bsz = 21,
random_state = random_state,
seqlen = seqlen,
)
with torch.autocast(device_type = "cuda", dtype = dtype):
for seqlen in [3341, 2048, 349]:
for random_state in [3407, 42]:
test_layernorm(
dim = dim,
eps = 1e-5,
dtype = dtype,
bsz = 21,
random_state = random_state,
seqlen = seqlen,
)
pass
pass
pass
pass