From 5c9629f5fe6df20de16facdf907fbd0666bfb21d Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sun, 10 Mar 2024 02:55:07 +1100 Subject: [PATCH] Update save.py --- unsloth/save.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 42d326e128..5971d76e6e 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -90,14 +90,14 @@ def _merge_lora(layer, name): W = fast_dequantize(W, quant_state) else: dtype = W.dtype - # W = W.to(torch.float32).t() - W = W.t() + W = W.to(torch.float32).t() + # W = W.t() if A is not None: # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32))) # W += sAB - # W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s) - W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s) + W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s) + # W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s) # if not torch.isfinite(W).all(): maximum_element = torch.max(W.min().abs(), W.max()) if not torch.isfinite(maximum_element).item():