From 213cfee903f57a185e718ba93038dced052ff993 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Wed, 7 Feb 2024 02:57:24 +1100 Subject: [PATCH] Update save.py --- unsloth/save.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index 41626cb4e8..60986a4af8 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -86,10 +86,10 @@ def _merge_lora(layer, name): W = W.to(torch.float32).t() if A is not None: - sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32))) + # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32))) # new_W = W + sAB - # W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s) - W += sAB + W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s) + # W += sAB # if not torch.isfinite(new_W).all() or not torch.isfinite(W).all(): # print(torch.dist(new_W, W)) # if not torch.isfinite(W).all():