From 28108608b8ff9dbbfb563c0bc22ffdef1f5e867b Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 19 Mar 2025 03:45:28 -0700 Subject: [PATCH] Update vision.py --- unsloth/models/vision.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 042281a1e8..38297c7667 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -169,7 +169,7 @@ def unsloth_base_fast_generate( autocaster = torch.autocast(device_type = "cuda", dtype = dtype) # Prepare LoRA - # state_dict = convert_lora_modules(self, dtype = dtype) + state_dict = convert_lora_modules(self, dtype = dtype) # Set compile dynamic shapes torch._dynamo.mark_static(input_ids, 0) @@ -202,16 +202,17 @@ def unsloth_base_fast_generate( kwargs["compile_config"] = _compile_config pass - with torch.inference_mode(), autocaster: - try: + try: + with torch.inference_mode(), autocaster: output = self._old_generate(*args, **kwargs) - except: - PROMPT_LOOPKUP[arch] = False - kwargs.pop("prompt_lookup_num_tokens", None) + except: + PROMPT_LOOPKUP[arch] = False + kwargs.pop("prompt_lookup_num_tokens", None) + with torch.inference_mode(), autocaster: output = self._old_generate(*args, **kwargs) - finally: - pass - # return_lora_modules(self, state_dict, torch.float32) + finally: + pass + return_lora_modules(self, state_dict, torch.float32) pass FastBaseModel.for_training(self)