From e6c3cdfc2d2dd206225a3879445dec00025235bb Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Sat, 9 Mar 2024 23:22:14 +1100 Subject: [PATCH] Update save.py --- unsloth/save.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unsloth/save.py b/unsloth/save.py index 5d1e07e18b..42d326e128 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -97,7 +97,7 @@ def _merge_lora(layer, name): # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32))) # W += sAB # W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s) - W.addmm_(A.t(), B.t(), alpha = s) + W.addmm_(A.t().to(W.dtype), B.t().to(W.dtype), alpha = s) # if not torch.isfinite(W).all(): maximum_element = torch.max(W.min().abs(), W.max()) if not torch.isfinite(maximum_element).item():