From 0d70391f9bcc8e5a14f8b8633e29e3014d862cd2 Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Mon, 8 Sep 2025 15:47:32 -0500 Subject: [PATCH] GptAttention turn training off during inference (#3289) --- unsloth/models/llama.py | 2 ++ unsloth/models/vision.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index f978060c9c..09bb3e04eb 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2986,6 +2986,7 @@ class FastLlamaModel: _for_inference(m) m = m.model _for_inference(m) + model.eval() # to turn off training on modules deeper in # Since transformers 4.53, must turn off explicitly for module in model.modules(): @@ -3030,6 +3031,7 @@ class FastLlamaModel: _for_training(m) m = m.model _for_training(m) + model.train() # to turn on training on modules deeper in # Since transformers 4.53, must turn on explicitly for module in model.modules(): diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index cce6554d52..3c71543a09 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -772,6 +772,7 @@ class FastBaseModel: _for_inference(m) m = m.model _for_inference(m) + model.eval() # to turn off training on modules deeper in # Since transformers 4.53, must turn off explicitly for module in model.modules(): @@ -823,6 +824,7 @@ class FastBaseModel: _for_training(m) m = m.model _for_training(m) + model.train() # to turn on training on modules deeper in # Since transformers 4.53, must turn on explicitly for module in model.modules():