From aa5668049a1c85b207daa1206da7b7770693de57 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 4 Nov 2024 23:19:02 -0800 Subject: [PATCH] Update rms_layernorm.py --- unsloth/kernels/rms_layernorm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unsloth/kernels/rms_layernorm.py b/unsloth/kernels/rms_layernorm.py index b6ffa5fee9..23b7342c87 100644 --- a/unsloth/kernels/rms_layernorm.py +++ b/unsloth/kernels/rms_layernorm.py @@ -144,7 +144,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function): n_rows, n_cols = X.shape BLOCK_SIZE : int num_warps : int - BLOCK_SIZE, num_warps = calculate_settings(n_cols) + # BLOCK_SIZE, num_warps = calculate_settings(n_cols) Y = torch.empty((n_rows, n_cols), dtype = X.dtype, device = "cuda:0") r = torch.empty(n_rows, dtype = torch.float32, device = "cuda:0") @@ -157,7 +157,7 @@ class Fast_RMS_Layernorm(torch.autograd.Function): r, r.stride(0), n_cols = int(n_cols), eps = float(eps), - BLOCK_SIZE = BLOCK_SIZE, + BLOCK_SIZE = 4096, num_warps = 16, ) else: