From 5c24ce4c81b978ba7456a934468f47a19bc62355 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 5 Feb 2025 06:02:42 -0800 Subject: [PATCH] Update utils.py --- unsloth/kernels/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py index 0bfd4269b1..f052914f98 100644 --- a/unsloth/kernels/utils.py +++ b/unsloth/kernels/utils.py @@ -124,6 +124,7 @@ global ABSMAX_BUFFER ABSMAX_BUFFER = None if HAS_CUDA_STREAM: + @torch.inference_mode def fast_dequantize(W, quant_state = None, out = None, use_global_buffer = False): if quant_state is None: return W if type(quant_state) is not list: @@ -193,6 +194,7 @@ if HAS_CUDA_STREAM: return out.t() if is_transposed else out pass else: + @torch.inference_mode def fast_dequantize(W, quant_state = None, out = None, use_global_buffer = False): if quant_state is None: return W if type(quant_state) is not list: