From 581235ba36ea4c45f1483f6f2b7654b3ba8a6014 Mon Sep 17 00:00:00 2001
From: Daniel Han-Chen <danielhanchen@gmail.com>
Date: Thu, 30 Nov 2023 23:07:19 +1100
Subject: [PATCH] Update fast_lora.py

---
 unsloth/kernels/fast_lora.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index 2e70444653..dcc887747f 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -43,7 +43,7 @@ def matmul_lora(X, W, W_quant, A, B, s, out = None):
 
     out = torch.matmul(X, W, out = out)
     if W_quant is not None: del W
-    out += (X @ A) @ (s * B)
+    out += (X @ A.to(dtype)) @ (s * B.to(dtype))
     return out.view(batch, seq_len, -1) if reshape else out
 pass