diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 93415f5647..59226f0f42 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -621,6 +621,7 @@ class FastModel(FastBaseModel):
                     "if hasattr(module, x): "\
                     "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\
                     "x = 'down_proj_bias'\n"\
+                    "if hasattr(module, x): "\
                     "setattr(module, x, torch.nn.Parameter(getattr(module, x).to(torch.float32)) if isinstance(getattr(module, x), torch.nn.Parameter) else getattr(module, x).to(torch.float32))\n"\
                     ";"
             else: