From c9150dcde759de697848062039cf8d6bde4cbb34 Mon Sep 17 00:00:00 2001 From: Datta Nimmaturi Date: Tue, 16 Sep 2025 17:59:08 +0530 Subject: [PATCH] Fast Inference with vLLM for VLMs (#2975) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [WIP] use vLLM for vision language models * Update README.md Editing icon sizes * Update README.md Updating icon sizes * Update README.md (#2885) * MoE kernels AGPLv3 * versioning * Many bug fixes (#2908) * add deepseek v3 * add deepseek r1 base * add deepseek r1 zero * add deepseek distill llama * add deepseek distill models * remove redundant code when constructing model names * add mistral small to registry * rename model registration methods * rename deepseek registration methods * refactor naming for mistral and phi * add global register models * refactor model registration tests for new registry apis * add model search method * remove deprecated registration api * add quant type test * add registry readme * make llama registration more specific * clear registry when executing individual model registration file * more registry readme updates * Update _auto_install.py * Llama4 * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Synthetic data * Update mapper.py * Xet and Synthetic * Update synthetic.py * Update loader.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py --------- Co-authored-by: jeromeku Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> * silienty skip falcon h1 import is transformers_version < 4.53.0 (#2912) * Dynamically adjust get_per_token_logps function and patch as well (#2911) * add intel gpu with vllm support (#2903) * [bugs] fix for casual mask (#2868) * fix for casual mask * use un_casual in sdpa * add missing mask * fix for type * Explicitly check if xformers exists for attention (#2889) * Update __init__.py * Update llama.py * if mlp doesn't exist in layer module check for feed_forward name for falcon h1 (#2913) * Move inputs to right devices. (#2919) * Move tensors to right devices * fix multi gpu for non mistral models * multi GPU RoPE for gemma2 * Finish up multi GPU inference * Make multiGPU rope a list * Remove unnecessary transfer to CPU * Remove unnecessary move to CPU * Donot move inputs to device yet will be handled separately in another PR * Move inputs to appropriate decoder device * Make device count global variable * Cleanup RoPE device code * Fixup num_gpu to device count * Cleanup device counts * Use device index for RoPE get_cache * Donot typecast * Use tuple instead of list for tensors. Use device index directly * fixup move to device logic * WIP VLM vLLM * Make vLLM patch a function * Add save and load lora functions * Make fast_inference setup depend on the flag * Improve fast inference patching mechanism * Make vision setting depend on checks in fastbasemodel * Check LoRA and vLLM intercompatibility for vision models * Comment pointing to vLLM LoRA check * Improve lora validation on vLLM * Error out on no vLLM and increase max lora rank * Bug fixes (#3017) * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Revert "Revert "Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model me…" (#2990) This reverts commit 663100749356af2539b03fc5c8984ebf183893ac. * skip_guard_eval_unsafe fix * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update llama.py * Update llama.py * Fix `quantization_method` * versioning * fix for casual mask (#3011) * [intel] add for intel path for llama.py (#3012) * fix for intel path * remove unuse code * Update unsloth/models/llama.py --------- Co-authored-by: Daniel Han * Update llama.py * Fix Gemma 2 (#3024) * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Revert "Revert "Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model me…" (#2990) This reverts commit 663100749356af2539b03fc5c8984ebf183893ac. * skip_guard_eval_unsafe fix * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update llama.py * Update llama.py * Fix `quantization_method` * versioning * Update _utils.py * Update _utils.py * Update _utils.py * falcon force float32 on sm<75 machines (#3026) * Fix torch compile issues (#3028) * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Revert "Revert "Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model me…" (#2990) This reverts commit 663100749356af2539b03fc5c8984ebf183893ac. * skip_guard_eval_unsafe fix * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update llama.py * Update llama.py * Fix `quantization_method` * versioning * Update _utils.py * Update _utils.py * Update _utils.py * check stride * Cleanup * Update rope_embedding.py * Update gemma2.py * Fix `set_stance` * Update pyproject.toml * Update _utils.py * Fixup patch vllm * Disable mllama * Use variables to decide VLM support * Better attn_impl handling * Patch TF protobuf incompatability * Torch 2.8 (#3186) * Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py * Update rl_replacements.py * Update rl.py * Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Torch 2.8 * Update rl_replacements.py --------- Co-authored-by: Datta Nimmaturi * Update _auto_install.py * Update pyproject.toml * Update rl.py * Protobuf issue * Update pyproject.toml * Fix extras transformers typo in pyproject.toml * Update _utils.py * Bug fixes (#3195) * Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py * Update rl_replacements.py * Update rl.py * Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Torch 2.8 * Update rl_replacements.py * Update loader.py * UNSLOTH_ENABLE_CCE * Fix * Update loader.py * Update loader.py * Update __init__.py * Update __init__.py * Update __init__.py * Update __init__.py * Import fixes * Update loader.py * Fix aimv2 issue * Update loader.py * Update import_fixes.py * Update import_fixes.py * Update loader.py * Update loader.py * Update loader.py * Upgrade * Update loader.py * Update loader.py * Update loader.py * Update loader.py --------- Co-authored-by: Datta Nimmaturi * adallow float32 dtype in FastLanguageModel (#3204) * Update loader.py * Update vision.py * Suppress message and use unsloth sampling params * Use trl sampling params for now * Improve error message * fixup quantized fast inference model name --------- Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Co-authored-by: Daniel Han Co-authored-by: jeromeku Co-authored-by: DoubleMathew Co-authored-by: Lei Zhenyuan Co-authored-by: parth2510 --- unsloth/models/_utils.py | 48 +++++++++++++ unsloth/models/llama.py | 27 +------- unsloth/models/loader.py | 58 ++++++++++------ unsloth/models/rl.py | 9 ++- unsloth/models/vision.py | 143 ++++++++++++++++++++++++++++++++++----- 5 files changed, 222 insertions(+), 63 deletions(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index a559d34ca4..bd3985e68d 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -69,6 +69,9 @@ __all__ = [ "patch_fast_lora", "validate_loftq_config", "RaiseUninitialized", + "fast_inference_setup", + "patch_peft_fast_inference", + "error_out_no_vllm", "dequantize_module_weight", ] @@ -191,6 +194,12 @@ if os.environ.get('UNSLOTH_ENABLE_LOGGING', '0') != '1': del vllm_block_pool_logger except: pass + try: + from vllm.lora.models import logger as vllm_lora_model_logger + vllm_lora_model_logger.addFilter(HideLoggingMessage("Regarding multimodal models, vLLM currently only supports adding")) + del vllm_lora_model_logger + except: + pass pass # The speedups for torchdynamo mostly come with GPU Ampere or higher and which is not detected here. @@ -1584,6 +1593,45 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m return loftq_config +def fast_inference_setup(model_name, model_config): + fast_inference = True + if not is_vLLM_available(): + logger.warning_once("Unsloth: vLLM is not installed! Will use Unsloth inference!") + fast_inference = False + pass + from unsloth_zoo.vllm_utils import ( + patch_vllm, + vllm_dynamic_quant_supported, + ) + patch_vllm() + if model_name.endswith("unsloth-bnb-4bit"): + if not vllm_dynamic_quant_supported(model_name, model_config): + # Instead use -bnb-4bit variant + logger.warning_once( + f"Unsloth: Switching from Unsloth dynamic quant to normal quant since\n"\ + f"we do not yet support fast inference for {model_name}" + ) + model_name = model_name[:-len("unsloth-bnb-4bit")] + "bnb-4bit" + pass + pass + return fast_inference, model_name + +def patch_peft_fast_inference(model): + vllm_engine = getattr(model.model, "vllm_engine", None) + if vllm_engine is not None: + model.vllm_engine = model.model.vllm_engine + model.fast_generate = model.model.fast_generate + model.fast_generate_batches = model.model.fast_generate_batches + + # Also saving and loading LoRA + from unsloth_zoo.vllm_utils import save_lora, load_lora + model.save_lora = functools.partial(save_lora, model) + model.load_lora = functools.partial(load_lora, model) + pass + +def error_out_no_vllm(*args, **kwargs): + raise NotImplementedError("Unsloth: vLLM is not yet supported for fast inference for this model! Please use `.generate` instead") + def _prepare_model_for_qat(model: torch.nn.Module, qat_scheme: str) -> torch.nn.Module: """ diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index f7a53d05fd..11e5eb359c 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2574,7 +2574,7 @@ class FastLlamaModel: raise NotImplementedError("Unsloth: Currently fast inference does not work with using biases for LoRA.") pass - #d oes not get lora yet, so get name from model, not base model + # Does not get lora yet, so get name from model, not base model is_classification = "Classification" in str(type(model)) arguments = dict( @@ -2694,17 +2694,7 @@ class FastLlamaModel: clean_gpu_cache() pass - # Patch for fast inference - if vllm_engine is not None: - model.vllm_engine = vllm_engine - model.fast_generate = vllm_fast_generate - model.fast_generate_batches = vllm_fast_generate_batches - - # Also saving and loading LoRA - from unsloth_zoo.vllm_utils import save_lora, load_lora - model.save_lora = functools.partial(save_lora, model) - model.load_lora = functools.partial(load_lora, model) - pass + patch_peft_fast_inference(model) # Add for_inference and for_training model.for_training = functools.partial(FastLlamaModel.for_training, model) @@ -2916,18 +2906,7 @@ class FastLlamaModel: clean_gpu_cache() pass - # Patch for fast inference - vllm_engine = getattr(model.model, "vllm_engine", None) - if vllm_engine is not None: - model.vllm_engine = model.model.vllm_engine - model.fast_generate = model.model.fast_generate - model.fast_generate_batches = model.model.fast_generate_batches - - # Also saving and loading LoRA - from unsloth_zoo.vllm_utils import save_lora, load_lora - model.save_lora = functools.partial(save_lora, model) - model.load_lora = functools.partial(load_lora, model) - pass + patch_peft_fast_inference(model) # Add for_inference and for_training model.for_training = functools.partial(FastLlamaModel.for_training, model) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index ab258f3ed9..ed973687d6 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -78,6 +78,7 @@ from ._utils import ( patch_compiled_autograd, process_vision_info, unsloth_compile_transformers, + fast_inference_setup, ) global FORCE_FLOAT32 @@ -142,6 +143,15 @@ class FastLanguageModel(FastLlamaModel): return_logits = False, # Return logits fullgraph = True, # No graph breaks use_exact_model_name = use_exact_model_name, + + # Pass vLLM/inference parameters + fast_inference = fast_inference, + gpu_memory_utilization = gpu_memory_utilization, + float8_kv_cache = float8_kv_cache, + random_state = random_state, + max_lora_rank = max_lora_rank, + disable_log_stats = disable_log_stats, + qat_scheme = qat_scheme, *args, **kwargs, ) @@ -370,6 +380,15 @@ class FastLanguageModel(FastLlamaModel): return_logits = False, # Return logits fullgraph = True, # No graph breaks use_exact_model_name = use_exact_model_name, + + # Pass vLLM/inference parameters + fast_inference = fast_inference, + gpu_memory_utilization = gpu_memory_utilization, + float8_kv_cache = float8_kv_cache, + random_state = random_state, + max_lora_rank = max_lora_rank, + disable_log_stats = disable_log_stats, + *args, **kwargs, ) pass @@ -388,26 +407,7 @@ class FastLanguageModel(FastLlamaModel): pass if fast_inference: - if not is_vLLM_available(): - print("Unsloth: vLLM is not installed! Will use Unsloth inference!") - fast_inference = False - pass - from unsloth_zoo.vllm_utils import ( - patch_vllm, - vllm_dynamic_quant_supported, - ) - patch_vllm() - if model_name.endswith("unsloth-bnb-4bit"): - if not vllm_dynamic_quant_supported(model_name, model_config): - # Instead use -bnb-4bit variant - print( - f"Unsloth: Switching from Unsloth dynamic quant to normal quant since\n"\ - f"we do not yet support fast inference for {model_name}" - ) - model_name = model_name[:-len("unsloth-bnb-4bit")] + "bnb-4bit" - pass - pass - pass + fast_inference, model_name = fast_inference_setup(model_name, model_config) model, tokenizer = dispatch_model.from_pretrained( model_name = model_name, @@ -530,6 +530,15 @@ class FastModel(FastBaseModel): whisper_language = None, whisper_task = None, unsloth_force_compile = False, + + # Add the missing vLLM/inference parameters + fast_inference = False, # uses vLLM + gpu_memory_utilization = 0.5, + float8_kv_cache = False, + random_state = 3407, + max_lora_rank = 64, + disable_log_stats = True, + qat_scheme = None, *args, **kwargs, ): @@ -884,6 +893,15 @@ class FastModel(FastBaseModel): supports_sdpa = supports_sdpa, whisper_language = whisper_language, whisper_task = whisper_task, + + # Pass vLLM/inference parameters + fast_inference = fast_inference, + gpu_memory_utilization = gpu_memory_utilization, + float8_kv_cache = float8_kv_cache, + random_state = random_state, + max_lora_rank = max_lora_rank, + disable_log_stats = disable_log_stats, + *args, **kwargs, ) diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 53f5eee66c..c5d30f1b89 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -550,7 +550,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): pass # Warn on too large or too small learning rate - if " learning_rate" in call_args: + if "learning_rate" in call_args: learning_rate_check = \ "if learning_rate < 1e-7: print(f'Unsloth: Your learning rate of `{learning_rate}` is too small and less than 1e-7! "\ "Consider increasing it, otherwise gradient updates will be close to 0!')\n"\ @@ -937,6 +937,13 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import r"\1, lora_request = self.model.load_lora('" + lora_name + r"', load_tensors = True))", source ) + # Prefer using unsloth's sampling params and fallback to trl's if not found + # We'll enable this later separately when combining both this and GRPOConfig params + # source = re.sub( + # r"sampling_params\s*=\s*sampling_params", + # r"sampling_params = getattr(self.args, 'vllm_sampling_params', sampling_params)", + # source + # ) # Skip if no changes done if source == original_source: continue diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 1451ed92cd..545a2d4d1d 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -75,6 +75,16 @@ __all__ = [ global NUM_LOGITS_TO_KEEP NUM_LOGITS_TO_KEEP = dict() +VLLM_SUPPORTED_VLM = [ + "qwen2_5_vl", + "gemma3", +] +VLLM_NON_LORA_VLM = [ + "mllama" +] + +from transformers import GenerationConfig, CompileConfig, HybridCache, AutoConfig, PretrainedConfig +HAS_TORCH_DTYPE = "torch_dtype" in PretrainedConfig.__doc__ from transformers import GenerationConfig, CompileConfig, HybridCache _compile_config = CompileConfig( @@ -254,8 +264,19 @@ class FastBaseModel: supports_sdpa = True, whisper_language = None, whisper_task = None, + fast_inference = False, + gpu_memory_utilization = 0.5, + float8_kv_cache = False, + random_state = 3407, + max_lora_rank = 64, + disable_log_stats = False, + unsloth_vllm_standby = False, **kwargs, ): + if unsloth_vllm_standby and os.environ.get("UNSLOTH_VLLM_STANDBY", "0") != "1": + raise RuntimeError("Unsloth: UNSLOTH_VLLM_STANDBY is True, but UNSLOTH_VLLM_STANDBY is not set to 1!") + pass + if model_types is None: raise RuntimeError( "Unsloth: Please use FastModel or FastVisionModel and not use FastBaseModel directly!" @@ -263,6 +284,31 @@ class FastBaseModel: if os.environ.get("UNSLOTH_MODEL_NAME", "") == "": os.environ["UNSLOTH_MODEL_NAME"] = model_name.lower() + is_vlm = (auto_model in [AutoModelForVision2Seq, AutoModelForImageTextToText]) + is_whisper = (whisper_language is not None and whisper_task is not None) + auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer + + model_type_arch = model_types[0] + if model_type_arch == "siglip": + for model_type_arch in model_types: + if model_type_arch != "siglip": break + + vllm_enable_lora = True + + if is_vlm and fast_inference: + if not any(arch in VLLM_SUPPORTED_VLM for arch in model_types): + raise RuntimeError( + f"Unsloth: Fast inference is only supported for Language models and Qwen2.5-VL, Gemma3 among vision models. " + f"Found architectures: {', '.join(model_types)}!" + ) + + if any(arch in VLLM_NON_LORA_VLM for arch in model_types): + # mllama is still only in vllm v0 https://arc.net/l/quote/llwkfgmu + # https://docs.vllm.ai/en/stable/models/supported_models.html#text-generation_1 + # vLLM V0 does not support LoRA on multi modal models. + # TODO: Update this once vLLM V1 supports Llama 3.2 aka mllama + vllm_enable_lora = False + os.environ["UNSLOTH_USE_NEW_MODEL"] = "1" if trust_remote_code: print( @@ -296,11 +342,6 @@ class FastBaseModel: max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) - model_type_arch = model_types[0] - if model_type_arch == "siglip": - for model_type_arch in model_types: - if model_type_arch != "siglip": break - statistics = \ f"==((====))== Unsloth {__version__}: Fast {model_type_arch.title()} patching. Transformers: {transformers_version}.{vllm_version}\n"\ f" {chr(92)}{chr(92)} /| {gpu_stats.name}. Num GPUs = {DEVICE_COUNT}. Max memory: {max_memory} GB. Platform: {platform_system}.\n"\ @@ -435,17 +476,68 @@ class FastBaseModel: kwargs = add_dtype_kwargs(torch_dtype, kwargs) raise_handler = RaiseUninitialized() - model = auto_model.from_pretrained( - model_name, - device_map = device_map, - # torch_dtype = torch_dtype, # Transformers removed torch_dtype - # quantization_config = bnb_config, - token = token, - trust_remote_code = trust_remote_code, - # attn_implementation = attn_implementation, - **kwargs, - ) + if not fast_inference: + model = auto_model.from_pretrained( + model_name, + device_map = device_map, + # torch_dtype = torch_dtype, # Transformers removed torch_dtype + # quantization_config = bnb_config, + token = token, + trust_remote_code = trust_remote_code, + # attn_implementation = attn_implementation, + **kwargs, + ) + model.fast_generate = model.generate + model.fast_generate_batches = error_out_no_vllm + else: + from unsloth_zoo.vllm_utils import ( + load_vllm, + get_vllm_state_dict, + convert_vllm_to_huggingface, + generate_batches, + ) + model_config = AutoConfig.from_pretrained( + model_name, + token = token, + attn_implementation = "sdpa" if supports_sdpa else "eager", + ) + + if fast_inference: + fast_inference, model_name = fast_inference_setup(model_name, model_config) + + allowed_args = inspect.getfullargspec(load_vllm).args + load_vllm_kwargs = dict( + model_name = model_name, + config = model_config, + gpu_memory_utilization = gpu_memory_utilization, + max_seq_length = max_seq_length, + dtype = dtype, + float8_kv_cache = float8_kv_cache, + enable_lora = vllm_enable_lora, + max_lora_rank = max_lora_rank, + disable_log_stats = disable_log_stats, + use_bitsandbytes = load_in_4bit, + unsloth_vllm_standby = unsloth_vllm_standby, + is_vision_model = is_vlm, + ) + for allowed_arg in allowed_args: + if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs: + load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg] + pass + + # Load vLLM first + llm = load_vllm(**load_vllm_kwargs) + + # Convert to HF format + _, quant_state_dict = get_vllm_state_dict(llm, config = model_config, is_vision_model = True) + model = convert_vllm_to_huggingface(quant_state_dict, model_config, dtype, bnb_config, is_vision_model = True) + model.vllm_engine = llm + model.fast_generate = model.vllm_engine.generate + model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine) + pass + raise_handler.remove() + # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer @@ -472,9 +564,6 @@ class FastBaseModel: # Counteract saved tokenizers tokenizer_name = model_name if tokenizer_name is None else tokenizer_name - is_vlm = (auto_model is AutoModelForVision2Seq) - is_whisper = (whisper_language is not None and whisper_task is not None) - auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer if (whisper_language and whisper_task) or auto_model.__name__.endswith("ForConditionalGeneration"): tokenizer = auto_processor.from_pretrained( tokenizer_name, @@ -627,6 +716,23 @@ class FastBaseModel: assert(type(target_modules) in (list, tuple, str,)) pass + if hasattr(model, "vllm_engine"): + if hasattr(model.vllm_engine, "llm_engine") and hasattr(model.vllm_engine.llm_engine, "vllm_config") and getattr(model.vllm_engine.llm_engine.vllm_config, "lora_config", None) is None: + # If vLLM is being used but lora is not enabled, throw an error + # Ref https://github.com/vllm-project/vllm/blob/51ba839555a5d122eadd91e9c16463ac288f5fa1/vllm/v1/engine/processor.py#L148-L151 + raise RuntimeError("Unsloth: LoRA is not enabled for this model!") + if finetune_vision_layers: + # vLLM does not support LoRA on vision layers + # https://github.com/vllm-project/vllm/blob/main/vllm/lora/models.py#L471-L477 + # TODO: Update this once vLLM V1 supports LoRA on vision layers (possibly not happening) + raise RuntimeError("Unsloth: Finetuning vision layers is not supported for fast_inference. Only text layers are supported!") + if model.config.model_type in VLLM_NON_LORA_VLM: + # mllama is still only in vllm v0 https://arc.net/l/quote/llwkfgmu + # https://docs.vllm.ai/en/stable/models/supported_models.html#text-generation_1 + # vLLM V0 does not support LoRA on multi modal models. + # TODO: Update this once vLLM V1 supports Llama 3.2 aka mllama + raise RuntimeError("Unsloth: LoRA finetuning for Llama 3.2 aka mllama models is not supported with fast_inference!") + # Clear deleted GPU items for _ in range(3): gc.collect() @@ -673,6 +779,7 @@ class FastBaseModel: torch.xpu.empty_cache() pass patch_saving_functions(model, vision = True) + patch_peft_fast_inference(model) # Add for_inference and for_training model.for_training = functools.partial(FastBaseModel.for_training, model)