From aa6ef77fad385a53df781a8d2a0c2783d92a2313 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 28 Nov 2024 00:01:25 -0800 Subject: [PATCH] skip modules --- unsloth/models/vision.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 80c1f82d4d..507e19cfe0 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -30,6 +30,7 @@ from transformers import set_seed as transformers_set_seed from unsloth_zoo.peft_utils import ( get_peft_regex, merge_and_overwrite_lora, + SKIP_QUANTIZATION_MODULES, ) from triton import __version__ as triton_version @@ -77,6 +78,7 @@ class FastBaseVisionModel: max_seq_length = None, dtype = None, load_in_4bit = True, + load_in_8bit token = None, device_map = "sequential", trust_remote_code = False, @@ -132,6 +134,7 @@ class FastBaseVisionModel: bnb_4bit_use_double_quant = True, bnb_4bit_quant_type = "nf4", bnb_4bit_compute_dtype = dtype, + llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES, ) pass @@ -424,5 +427,3 @@ class FastBaseVisionModel: return model pass pass - -