From 53ca91f74439b01169d77e449557044d5114c068 Mon Sep 17 00:00:00 2001 From: Daniel Han-Chen Date: Tue, 6 Feb 2024 01:30:48 +1100 Subject: [PATCH] Update save.py --- unsloth/save.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unsloth/save.py b/unsloth/save.py index efffbf5979..1914d0f983 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -94,13 +94,13 @@ def _merge_lora(layer, name, max_vram): W = W.to(torch.float32).t() if A is not None: - # sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32))) - # W += sAB if out_of_memory: A = A.to("cpu", non_blocking = True, copy = True) B = B.to("cpu", non_blocking = True, copy = True) pass - W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s) + # W.addmm_(A.t().to(torch.float32), B.t().to(torch.float32), alpha = s) + sAB = (A.t().to(torch.float32) @ (s * B.t().to(torch.float32))) + W += sAB # if not torch.isfinite(W).all(): maximum_element = torch.max(W.min().abs(), W.max()) if not torch.isfinite(maximum_element).item():