diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 63f48af659..747858d011 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.5.5" +__version__ = "2025.5.6" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 9c5a7b68be..8a49026984 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -461,6 +461,12 @@ except: from transformers import AutoModelForVision2Seq pass +DISABLE_COMPILE_MODEL_NAMES = [ + "aya-vision", + "modernbert", + "granite-vision", +] + class FastModel(FastBaseModel): @staticmethod @@ -521,33 +527,35 @@ class FastModel(FastBaseModel): model_name = get_model_name(model_name, load_in_4bit) # Check versions + lowered_model_name = model_name.lower() LATEST = '\nPlease use transformers via `pip install --no-deps git+https://github.com/huggingface/transformers.git`' NIGHTLY = '\nPlease use nightly transformers via pip install --upgrade "transformers>=4.49.0"`' - if "pixtral" in model_name.lower() and transformers_version < Version("4.49.0"): + if "pixtral" in lowered_model_name and transformers_version < Version("4.49.0"): raise RuntimeError("Unsloth: Pixtral only works on transformers >= 4.49.0." + LATEST) - elif "qwen2.5" in model_name.lower() and transformers_version < Version("4.49.0"): + elif "qwen2.5" in lowered_model_name and transformers_version < Version("4.49.0"): raise RuntimeError("Unsloth: Qwen 2.5 only works on transformers >= 4.49.0." + LATEST) - elif "aya-vision" in model_name.lower(): - # Disable compiling for now - errors out! - os.environ["UNSLOTH_COMPILE_DISABLE"] = "1" - if transformers_version < Version("4.50.0.dev0"): - raise RuntimeError("Unsloth: Aya Vision only works on transformers >= 4.50.0." + NIGHTLY) - elif "gemma-3" in model_name.lower() and transformers_version < Version("4.50.0.dev0"): + elif "gemma-3" in lowered_model_name and transformers_version < Version("4.50.0.dev0"): raise RuntimeError("Unsloth: Gemma 3 only works on transformers >= 4.50.0." + NIGHTLY) - elif "c4ai-command-a-03-2025" in model_name.lower() and transformers_version < Version("4.50.0.dev0"): + elif "c4ai-command-a-03-2025" in lowered_model_name and transformers_version < Version("4.50.0.dev0"): raise RuntimeError("Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY) - elif "granite-vision" in model_name.lower(): - # Disable compiling for now - errors out! - os.environ["UNSLOTH_COMPILE_DISABLE"] = "1" - if transformers_version < Version("4.50.0.dev0"): - raise RuntimeError("Unsloth: Granite Vision only works on transformers >= 4.50.0." + NIGHTLY) - elif "csm-1b" in model_name.lower(): + elif "csm-1b" in lowered_model_name: os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = "torch.float16;if name.endswith(('_proj', 'fc1', 'fc2', 'codebook', 'head')): module.to(torch.float16)" - elif "olmo-2" in model_name.lower() and transformers_version < Version("4.50.0.dev0"): + elif "olmo-2" in lowered_model_name and transformers_version < Version("4.50.0.dev0"): raise RuntimeError("Unsloth: OLMo-2 only works on transformers >= 4.50.0." + NIGHTLY) - elif "whisper" in model_name.lower(): - os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Whisper fails + else: + for check_model_name in DISABLE_COMPILE_MODEL_NAMES: + if check_model_name in lowered_model_name: + os.environ["UNSLOTH_COMPILE_DISABLE"] = "1" + os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" + if transformers_version < Version("4.50.0.dev0"): + raise RuntimeError(f"Unsloth: {check_model_name} only works on transformers >= 4.50.0." + NIGHTLY) + break + pass + + if auto_model is not None: + # All other models need to disable static cache + os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" pass if USE_MODELSCOPE and not os.path.exists(model_name): diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 4bbd8295cc..e50a5a877d 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -728,16 +728,6 @@ __INT_TO_FLOAT_MAPPER = \ "mistralai/Mistral-Small-3.1-24B-Base-2503", "unsloth/Mistral-Small-3.1-24B-Base-2503-bnb-4bit", ), - "unsloth/orpheus-3b-0.1-pretrained-unsloth-bnb-4bit" : ( - "unsloth/orpheus-3b-0.1-pretrained", - "canopylabs/orpheus-3b-0.1-pretrained", - "unsloth/orpheus-3b-0.1-pretrained-bnb-4bit", - ), - "unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit" : ( - "unsloth/orpheus-3b-0.1-ft", - "canopylabs/orpheus-3b-0.1-ft", - "unsloth/orpheus-3b-0.1-ft-bnb-4bit", - ), "unsloth/Qwen3-0.6B-unsloth-bnb-4bit" : ( "unsloth/Qwen3-0.6B", "Qwen/Qwen3-0.6B", @@ -817,6 +807,16 @@ __INT_TO_FLOAT_MAPPER = \ "microsoft/Phi-4-mini-reasoning", "unsloth/phi-4-mini-reasoning-bnb-4bit", ), + "unsloth/orpheus-3b-0.1-pretrained-unsloth-bnb-4bit" : ( + "unsloth/orpheus-3b-0.1-pretrained", + "canopylabs/orpheus-3b-0.1-pretrained", + "unsloth/orpheus-3b-0.1-pretrained-bnb-4bit", + ), + "unsloth/orpheus-3b-0.1-ft-unsloth-bnb-4bit" : ( + "unsloth/orpheus-3b-0.1-ft", + "canopylabs/orpheus-3b-0.1-ft", + "unsloth/orpheus-3b-0.1-ft-bnb-4bit", + ), "unsloth/csm-1b" : ( "unsloth/csm-1b", "sesame/csm-1b", @@ -837,6 +837,18 @@ __INT_TO_FLOAT_MAPPER = \ "unsloth/CrisperWhisper", "nyrahealth/CrisperWhisper", ), + "unsloth/Llasa-1B" : ( + "unsloth/Llasa-1B", + "HKUSTAudio/Llasa-1B", + ), + "unsloth/Spark-TTS-0.5B" : ( + "unsloth/Spark-TTS-0.5B", + "SparkAudio/Spark-TTS-0.5B", + ), + "unsloth/Llama-OuteTTS-1.0-1B" : ( + "unsloth/Llama-OuteTTS-1.0-1B", + "OuteAI/Llama-OuteTTS-1.0-1B", + ), } INT_TO_FLOAT_MAPPER = {} diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 2bff87d8d9..4466128a28 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -95,8 +95,18 @@ def unsloth_base_fast_generate( input_ids = kwargs["input_ids"] elif "input" in kwargs: input_ids = kwargs["input_ids"] + elif "input_features" in kwargs: + input_ids = kwargs["input_features"] + elif "input_embeds" in kwargs: + input_ids = kwargs["input_embeds"] + elif "inputs" in kwargs: + input_ids = kwargs["inputs"] else: - raise TypeError("Unsloth: You need to pass in input_ids to .generate!") + key = next(iter(kwargs.keys())) + if type(kwargs["key"]) is not torch.Tensor: + raise TypeError("Unsloth: You need to pass in input_ids to .generate!") + input_ids = kwargs[key] + pass assert(type(input_ids) is torch.Tensor) bsz = input_ids.shape[0] @@ -203,10 +213,11 @@ def unsloth_base_fast_generate( if "generation_config" in kwargs: kwargs["generation_config"].cache_implementation = cache_implementation - kwargs["generation_config"].compile_config = _compile_config if cache_implementation is not None else None + if cache_implementation is not None: + kwargs["generation_config"].compile_config = _compile_config else: kwargs["cache_implementation"] = cache_implementation - if cache_implementation: + if cache_implementation is not None: kwargs["compile_config"] = _compile_config pass