From 19fb50e24498086cbae0fcc942616630779face4 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Thu, 1 Feb 2024 19:56:54 +1100 Subject: [PATCH] Update utils.py --- unsloth/kernels/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index de97fea729..4f4ce41006 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -119,7 +119,7 @@ def fast_gemv(X, W, quant_state, out = None): # For fast X @ W where seq_len == 1 # From https://github.com/TimDettmers/bitsandbytes/blob/main/bitsandbytes/functional.py#L1469 bsz, q_len, hd = X.shape - # assert(q_len == 1) + assert(q_len == 1) if type(quant_state) is not list: # https://github.com/TimDettmers/bitsandbytes/pull/763/files @@ -138,7 +138,7 @@ def fast_gemv(X, W, quant_state, out = None): offset, state2 = compressed_stats absmax2, code2, blocksize2, _, _, _, _ = state2 pass - # assert(dtype == X.dtype) + assert(dtype == X.dtype) bout = shape[0] if out is None: @@ -152,7 +152,7 @@ def fast_gemv(X, W, quant_state, out = None): k = shape[1] lda = shape[0] ldc = shape[0] - ldb = (hd+1)//2 + ldb = (X.shape[-1]+1)//2 m = ctypes.c_int32(m) n = ctypes.c_int32(n) k = ctypes.c_int32(k)