From 4f8027ad741520f5a434bf021cda54e1483ba385 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 15 May 2025 02:32:29 -0700 Subject: [PATCH] is_multimodal --- unsloth/models/loader.py | 5 +++-- unsloth/models/vision.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 6497887b72..8e986f0a9a 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -485,6 +485,7 @@ class FastModel(FastBaseModel): auto_model = None, whisper_language = None, whisper_task = None, + is_multimodal = None, *args, **kwargs, ): if token is None: token = get_token() @@ -541,8 +542,8 @@ class FastModel(FastBaseModel): if transformers_version < Version("4.50.0.dev0"): raise RuntimeError("Unsloth: Granite Vision only works on transformers >= 4.50.0." + NIGHTLY) elif "csm-1b" in model_name.lower(): - os.environ["UNSLOTH_COMPILE_DISABLE"] = "1" - os.environ["UNSLOTH_DISABLE_FAST_GENERATION"] = "0" + os.environ["UNSLOTH_COMPILE_DISABLE"] = "0" + os.environ["UNSLOTH_DISABLE_FAST_GENERATION"] = "1" os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = "torch.float16;if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)" elif "olmo-2" in model_name.lower() and transformers_version < Version("4.50.0.dev0"): raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 94e2b4c3d4..31a8802e21 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -243,6 +243,7 @@ class FastBaseModel: supports_sdpa = True, whisper_language = None, whisper_task = None, + is_multimodal = False, **kwargs, ): if model_types is None: @@ -398,7 +399,7 @@ class FastBaseModel: is_vlm = (auto_model is AutoModelForVision2Seq) is_whisper = (whisper_language is not None and whisper_task is not None) auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer - if whisper_language and whisper_task: + if (whisper_language and whisper_task) or is_multimodal: tokenizer = auto_processor.from_pretrained( tokenizer_name, padding_side = "right",