Bug fixes (#3484)
* Update loader.py * Update import_fixes.py * Update import_fixes.py * Update loader.py * Update loader.py * Update loader.py * Upgrade * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * custom_datatype * recheck * Float16 * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Bug fix * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * torch_dtype * Update rl.py * Fix CE Loss * Versioning * Update loader.py * Update loader.py * extract_model_type_from_config * Model types * Update loader.py * get_transformers_model_type * Update loader.py * Update loader.py * Update loader.py * Update rl.py * Update pyproject.toml * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Versioning * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update vision.py * Update vision.py * Fix DataParallel * Update _utils.py * Update rl.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update mapper.py * Versioning * Update loader.py * Update loader.py * Update rl.py * Versioning * Update _utils.py * Fix auto_mapping * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Message * Update vision.py * Update loader.py * Update vision.py * cache_implementation * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Save max_seq_length * Update _utils.py * Update rl.py * Update vision.py * Update llama.py * Mistral3 vllm (#3349) * [WIP] use vLLM for vision language models * Update README.md Editing icon sizes * Update README.md Updating icon sizes * Update README.md (#2885) * MoE kernels AGPLv3 * versioning * Many bug fixes (#2908) * add deepseek v3 * add deepseek r1 base * add deepseek r1 zero * add deepseek distill llama * add deepseek distill models * remove redundant code when constructing model names * add mistral small to registry * rename model registration methods * rename deepseek registration methods * refactor naming for mistral and phi * add global register models * refactor model registration tests for new registry apis * add model search method * remove deprecated registration api * add quant type test * add registry readme * make llama registration more specific * clear registry when executing individual model registration file * more registry readme updates * Update _auto_install.py * Llama4 * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Synthetic data * Update mapper.py * Xet and Synthetic * Update synthetic.py * Update loader.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py --------- Co-authored-by: jeromeku <jerome.ku@gmail.com> Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> * silienty skip falcon h1 import is transformers_version < 4.53.0 (#2912) * Dynamically adjust get_per_token_logps function and patch as well (#2911) * add intel gpu with vllm support (#2903) * [bugs] fix for casual mask (#2868) * fix for casual mask * use un_casual in sdpa * add missing mask * fix for type * Explicitly check if xformers exists for attention (#2889) * Update __init__.py * Update llama.py * if mlp doesn't exist in layer module check for feed_forward name for falcon h1 (#2913) * Move inputs to right devices. (#2919) * Move tensors to right devices * fix multi gpu for non mistral models * multi GPU RoPE for gemma2 * Finish up multi GPU inference * Make multiGPU rope a list * Remove unnecessary transfer to CPU * Remove unnecessary move to CPU * Donot move inputs to device yet will be handled separately in another PR * Move inputs to appropriate decoder device * Make device count global variable * Cleanup RoPE device code * Fixup num_gpu to device count * Cleanup device counts * Use device index for RoPE get_cache * Donot typecast * Use tuple instead of list for tensors. Use device index directly * fixup move to device logic * WIP VLM vLLM * Make vLLM patch a function * Add save and load lora functions * Make fast_inference setup depend on the flag * Improve fast inference patching mechanism * Make vision setting depend on checks in fastbasemodel * Check LoRA and vLLM intercompatibility for vision models * Comment pointing to vLLM LoRA check * Improve lora validation on vLLM * Error out on no vLLM and increase max lora rank * Bug fixes (#3017) * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Revert "Revert "Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model me…" (#2990) This reverts commit6631007493. * skip_guard_eval_unsafe fix * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update llama.py * Update llama.py * Fix `quantization_method` * versioning * fix for casual mask (#3011) * [intel] add for intel path for llama.py (#3012) * fix for intel path * remove unuse code * Update unsloth/models/llama.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Update llama.py * Fix Gemma 2 (#3024) * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Revert "Revert "Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model me…" (#2990) This reverts commit6631007493. * skip_guard_eval_unsafe fix * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update llama.py * Update llama.py * Fix `quantization_method` * versioning * Update _utils.py * Update _utils.py * Update _utils.py * falcon force float32 on sm<75 machines (#3026) * Fix torch compile issues (#3028) * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Revert "Revert "Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model me…" (#2990) This reverts commit6631007493. * skip_guard_eval_unsafe fix * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update llama.py * Update llama.py * Fix `quantization_method` * versioning * Update _utils.py * Update _utils.py * Update _utils.py * check stride * Cleanup * Update rope_embedding.py * Update gemma2.py * Fix `set_stance` * Update pyproject.toml * Update _utils.py * Fixup patch vllm * Disable mllama * Use variables to decide VLM support * Better attn_impl handling * Patch TF protobuf incompatability * Torch 2.8 (#3186) * Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py * Update rl_replacements.py * Update rl.py * Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Torch 2.8 * Update rl_replacements.py --------- Co-authored-by: Datta Nimmaturi <venkatadattasainimmaturi@gmail.com> * Update _auto_install.py * Update pyproject.toml * Update rl.py * Protobuf issue * Update pyproject.toml * Fix extras transformers typo in pyproject.toml * Update _utils.py * Bug fixes (#3195) * Fix mamba * Update loader.py * Update vision.py * Update loader.py * Filter vLLM standby logs (#3131) * filter vLLM standby logs * safeguard standby logger patch * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py * Update unsloth/models/_utils.py --------- Co-authored-by: Daniel Han <danielhanchen@gmail.com> * Update loader.py * Add scaler * Update llama.py * Update _utils.py * Versioning * GPT OSS fix * GPT OSS fix * Update loader.py * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Update vision.py * Update llama.py * Update llama.py * Update llama.py * Versioning * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Upcast norms * Update loader.py * Update vision.py * Upcast layernorms * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update save.py * Update rl.py * Update pyproject.toml * Update rl.py * Update rl_replacements.py * Update rl.py * Update rl.py * Update rl.py * Update _utils.py * Update __init__.py * Torch 2.8 * Update rl_replacements.py * Update loader.py * UNSLOTH_ENABLE_CCE * Fix * Update loader.py * Update loader.py * Update __init__.py * Update __init__.py * Update __init__.py * Update __init__.py * Import fixes * Update loader.py * Fix aimv2 issue * Update loader.py * Update import_fixes.py * Update import_fixes.py * Update loader.py * Update loader.py * Update loader.py * Upgrade * Update loader.py * Update loader.py * Update loader.py * Update loader.py --------- Co-authored-by: Datta Nimmaturi <venkatadattasainimmaturi@gmail.com> * adallow float32 dtype in FastLanguageModel (#3204) * Update loader.py * Update vision.py * Suppress message and use unsloth sampling params * Use trl sampling params for now * Improve error message * fixup quantized fast inference model name * Add mistral 3 support --------- Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Co-authored-by: Daniel Han <danielhanchen@gmail.com> Co-authored-by: jeromeku <jerome.ku@gmail.com> Co-authored-by: DoubleMathew <mmathew23@gmail.com> Co-authored-by: Lei Zhenyuan <zhenyuan.lei@intel.com> Co-authored-by: parth2510 <parthguptapg7326@gmail.com> * Set padding to 0 * Fix patch * fixup patch (#3359) Co-authored-by: Datta Nimmaturi <venkatadattasainimmaturi@gmail.com> * Update vision.py * Versioning * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * MXFP4 dequant * Update loader.py * Update vision.py * load_in_16bit * Update vision.py * Update vision.py * Update vision.py * Update rl.py * Update vision.py * offload_embedding * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update rl_replacements.py * Update loader.py * Fix padding issue * Update pyproject.toml * Update _utils.py * Update pyproject.toml * Update _utils.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * New models * Update llama.py * Versioning * Update _utils.py * Update llama.py * Update _utils.py * Update llama.py * Fix AMD * Update _utils.py * Update llama.py * Update vision.py * DEVICE_TYPE_TORCH * Update __init__.py * Update __init__.py * Update _utils.py * Move DEVICE_TYPE * Update rl_replacements.py * Update loader.py * AMD install script * Move AMD * Update _amd_install.sh * Update pyproject.toml * Update pyproject.toml * Delete _amd_install.sh * Update device_type.py * Update loader.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update tokenizer_utils.py * Versioning * Update pyproject.toml * Update loader.py * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update _utils.py * Update pyproject.toml * Update _utils.py * Update _utils.py * Update loader.py * Update _utils.py * Update _utils.py * local_files_only * Cut Cross Entropy * Update llama.py --------- Co-authored-by: Datta Nimmaturi <venkatadattasainimmaturi@gmail.com> Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Co-authored-by: jeromeku <jerome.ku@gmail.com> Co-authored-by: DoubleMathew <mmathew23@gmail.com> Co-authored-by: Lei Zhenyuan <zhenyuan.lei@intel.com> Co-authored-by: parth2510 <parthguptapg7326@gmail.com>
This commit is contained in:
Daniel Han
GitHub
parent
ea8e564cf0
commit
c33f2cee15
5 changed files with 146 additions and 99 deletions
|
|
@ -905,6 +905,26 @@ if Version(peft_version) < Version("0.12.0"):
|
|||
pass
|
||||
|
||||
# =============================================
|
||||
import importlib
|
||||
global USE_MODELSCOPE
|
||||
USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1"
|
||||
if USE_MODELSCOPE:
|
||||
if importlib.util.find_spec("modelscope") is None:
|
||||
raise ImportError(f'You are using the modelscope hub, please install modelscope by `pip install modelscope -U`')
|
||||
pass
|
||||
pass
|
||||
|
||||
import socket
|
||||
@functools.lru_cache(1)
|
||||
def has_internet(host = "8.8.8.8", port = 53, timeout = 3):
|
||||
if os.environ.get("TRANSFORMERS_OFFLINE", "0") == "1": return False
|
||||
try:
|
||||
socket.setdefaulttimeout(timeout)
|
||||
socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect((host, port))
|
||||
return True
|
||||
except socket.error as ex:
|
||||
return False
|
||||
pass
|
||||
|
||||
import psutil
|
||||
def _get_statistics(statistics = None, force_download = True):
|
||||
|
|
@ -912,61 +932,81 @@ def _get_statistics(statistics = None, force_download = True):
|
|||
# We simply download a README.md file from HF - all data is made public.
|
||||
# This is simply so we can check if some envs are broken or not.
|
||||
# You can disable this by commenting the below out
|
||||
try:
|
||||
n_cpus = psutil.cpu_count(logical = False)
|
||||
keynames = "\n" + "\n".join(os.environ.keys())
|
||||
if statistics is not None: pass
|
||||
elif "\nCOLAB_" in keynames and n_cpus == 1: statistics = "colab"
|
||||
elif "\nCOLAB_" in keynames: statistics = "colabpro"
|
||||
elif "\nKAGGLE_" in keynames: statistics = "kaggle"
|
||||
elif "\nRUNPOD_" in keynames: statistics = "runpod"
|
||||
elif "\nAWS_" in keynames: statistics = "aws"
|
||||
elif "\nAZURE_" in keynames: statistics = "azure"
|
||||
# elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp"
|
||||
elif "\nINVOCATION_ID" in keynames: statistics = "lambda"
|
||||
# else: statistics = "other"
|
||||
else:
|
||||
def try_vllm_check():
|
||||
vendor_files = (
|
||||
"/sys/class/dmi/id/product_version",
|
||||
"/sys/class/dmi/id/bios_vendor",
|
||||
"/sys/class/dmi/id/product_name",
|
||||
"/sys/class/dmi/id/chassis_asset_tag",
|
||||
"/sys/class/dmi/id/sys_vendor",
|
||||
)
|
||||
from pathlib import Path
|
||||
for vendor_file in vendor_files:
|
||||
path = Path(vendor_file)
|
||||
if path.is_file():
|
||||
file_content = path.read_text().lower()
|
||||
if "amazon" in file_content: return "aws"
|
||||
elif "microsoft corporation" in file_content: return "azure"
|
||||
elif "google" in file_content: return "gcp"
|
||||
return "other"
|
||||
pass
|
||||
try: statistics = try_vllm_check()
|
||||
except: statistics = "other"
|
||||
pass
|
||||
if statistics is not None:
|
||||
from transformers import AutoModelForCausalLM
|
||||
stats_model = AutoModelForCausalLM.from_pretrained(
|
||||
f"unslothai/{statistics}",
|
||||
force_download = force_download,
|
||||
n_cpus = psutil.cpu_count(logical = False)
|
||||
keynames = "\n" + "\n".join(os.environ.keys())
|
||||
# Check modelscope for down detection
|
||||
global USE_MODELSCOPE
|
||||
USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1"
|
||||
|
||||
if statistics is not None: pass
|
||||
elif "\nCOLAB_" in keynames and n_cpus == 1: statistics = "colab"
|
||||
elif "\nCOLAB_" in keynames: statistics = "colabpro"
|
||||
elif "\nKAGGLE_" in keynames: statistics = "kaggle"
|
||||
elif "\nRUNPOD_" in keynames: statistics = "runpod"
|
||||
elif "\nAWS_" in keynames: statistics = "aws"
|
||||
elif "\nAZURE_" in keynames: statistics = "azure"
|
||||
# elif "\nK_" in keynames or "\nFUNCTION_" in keynames: statistics = "gcp"
|
||||
elif "\nINVOCATION_ID" in keynames: statistics = "lambda"
|
||||
# else: statistics = "other"
|
||||
else:
|
||||
def try_vllm_check():
|
||||
vendor_files = (
|
||||
"/sys/class/dmi/id/product_version",
|
||||
"/sys/class/dmi/id/bios_vendor",
|
||||
"/sys/class/dmi/id/product_name",
|
||||
"/sys/class/dmi/id/chassis_asset_tag",
|
||||
"/sys/class/dmi/id/sys_vendor",
|
||||
)
|
||||
del stats_model
|
||||
from pathlib import Path
|
||||
for vendor_file in vendor_files:
|
||||
path = Path(vendor_file)
|
||||
if path.is_file():
|
||||
file_content = path.read_text().lower()
|
||||
if "amazon" in file_content: return "aws"
|
||||
elif "microsoft corporation" in file_content: return "azure"
|
||||
elif "google" in file_content: return "gcp"
|
||||
return "other"
|
||||
pass
|
||||
except:
|
||||
try: statistics = try_vllm_check()
|
||||
except: statistics = "other"
|
||||
pass
|
||||
if statistics is not None:
|
||||
import tempfile
|
||||
from huggingface_hub import snapshot_download
|
||||
from unsloth_zoo.rl_environments import execute_with_time_limit
|
||||
if has_internet():
|
||||
@execute_with_time_limit(120)
|
||||
def stats_check():
|
||||
with tempfile.TemporaryDirectory(ignore_cleanup_errors = True) as f:
|
||||
snapshot_download(f"unslothai/{statistics}", force_download = True, cache_dir = f, local_dir = f)
|
||||
try:
|
||||
stats_check()
|
||||
except TimeoutError:
|
||||
raise TimeoutError(
|
||||
"Unsloth: HuggingFace seems to be down after trying for 120 seconds :(\n"\
|
||||
"Check https://status.huggingface.co/ for more details.\n"\
|
||||
"As a temporary measure, use modelscope with the same model name ie:\n"\
|
||||
"```\n"\
|
||||
"pip install modelscope\n"\
|
||||
"import os; os.environ['UNSLOTH_USE_MODELSCOPE'] = '1'\n"\
|
||||
"from unsloth import FastLanguageModel\n"\
|
||||
"model = FastLanguageModel.from_pretrained('unsloth/gpt-oss-20b')\n"\
|
||||
"```"
|
||||
)
|
||||
pass
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
def get_statistics():
|
||||
def get_statistics(local_files_only = False):
|
||||
# We log some basic stats about which environment is being used.
|
||||
# This is also to check if HuggingFace is down or not!
|
||||
# We simply download a README.md file from HF - all data is made public.
|
||||
# This is simply so we can check if some envs are broken or not.
|
||||
# You can disable this by setting UNSLOTH_DISABLE_STATISTICS
|
||||
import os
|
||||
if "UNSLOTH_DISABLE_STATISTICS" in os.environ: return
|
||||
if local_files_only: return
|
||||
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled
|
||||
disabled = False
|
||||
if not are_progress_bars_disabled():
|
||||
|
|
@ -975,24 +1015,17 @@ def get_statistics():
|
|||
pass
|
||||
_get_statistics(None)
|
||||
_get_statistics("repeat", force_download = False)
|
||||
try:
|
||||
vram = torch.cuda.get_device_properties(0).total_memory / 1024 / 1024 / 1024
|
||||
if vram <= 8 : vram = 8
|
||||
elif vram <= 16: vram = 16
|
||||
elif vram <= 20: vram = 20
|
||||
elif vram <= 24: vram = 24
|
||||
elif vram <= 40: vram = 40
|
||||
elif vram <= 48: vram = 48
|
||||
elif vram <= 80: vram = 80
|
||||
else: vram = 96
|
||||
_get_statistics(f"vram-{vram}")
|
||||
except:
|
||||
pass
|
||||
pass
|
||||
try:
|
||||
_get_statistics(f"{DEVICE_COUNT if DEVICE_COUNT <= 8 else 9}")
|
||||
except:
|
||||
pass
|
||||
vram = torch.cuda.get_device_properties(0).total_memory / 1024 / 1024 / 1024
|
||||
if vram <= 8 : vram = 8
|
||||
elif vram <= 16: vram = 16
|
||||
elif vram <= 20: vram = 20
|
||||
elif vram <= 24: vram = 24
|
||||
elif vram <= 40: vram = 40
|
||||
elif vram <= 48: vram = 48
|
||||
elif vram <= 80: vram = 80
|
||||
else: vram = 96
|
||||
_get_statistics(f"vram-{vram}")
|
||||
_get_statistics(f"{DEVICE_COUNT if DEVICE_COUNT <= 8 else 9}")
|
||||
if disabled: enable_progress_bars()
|
||||
pass
|
||||
|
||||
|
|
@ -1592,14 +1625,6 @@ for j, function in enumerate(functions):
|
|||
except: continue
|
||||
pass
|
||||
|
||||
import importlib
|
||||
USE_MODELSCOPE = os.environ.get("UNSLOTH_USE_MODELSCOPE", "0") == "1"
|
||||
if USE_MODELSCOPE:
|
||||
if importlib.util.find_spec("modelscope") is None:
|
||||
raise ImportError(f'You are using the modelscope hub, please install modelscope by `pip install modelscope -U`')
|
||||
pass
|
||||
pass
|
||||
|
||||
|
||||
def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, model):
|
||||
from peft import LoraConfig
|
||||
|
|
|
|||
|
|
@ -1205,7 +1205,7 @@ def CausalLM_fast_forward(fast_forward_inference):
|
|||
# < 1024 Normal Unsloth uses less VRAM!
|
||||
if DEVICE_TYPE == "hip":
|
||||
# [TODO] AMD GPUs fail on chunked_cross_entropy loss!
|
||||
# RuntimeError: Triton Error [HIP]: Code: 1, Messsage: invalid argument
|
||||
# RuntimeError: Triton Error [HIP]: Code: 1, Messsage: invalid argument
|
||||
RETURN_LOGITS = False
|
||||
elif bsz*q_len <= 1024:
|
||||
RETURN_LOGITS = True
|
||||
|
|
@ -1217,6 +1217,8 @@ def CausalLM_fast_forward(fast_forward_inference):
|
|||
if self.config.model_type == "falcon_h1":
|
||||
hidden_states = hidden_states * self.config.lm_head_multiplier
|
||||
|
||||
### DISABLED since T4 breaks
|
||||
# OutOfResources: out of resource: shared memory, Required: 98304, Hardware limit: 65536. Reducing block sizes or `num_stages` may help.
|
||||
# loss = fused_linear_cross_entropy(
|
||||
# hidden_states = hidden_states,
|
||||
# lm_weight = lm_head,
|
||||
|
|
@ -1242,11 +1244,11 @@ def CausalLM_fast_forward(fast_forward_inference):
|
|||
return (loss,) + output if loss is not None else output
|
||||
|
||||
output = CausalLMOutputWithPast(
|
||||
loss=loss,
|
||||
logits=EMPTY_LOGITS,
|
||||
past_key_values=outputs.past_key_values,
|
||||
hidden_states=outputs.hidden_states,
|
||||
attentions=outputs.attentions,
|
||||
loss = loss,
|
||||
logits = EMPTY_LOGITS,
|
||||
past_key_values= outputs.past_key_values,
|
||||
hidden_states = outputs.hidden_states,
|
||||
attentions = outputs.attentions,
|
||||
)
|
||||
return output
|
||||
pass
|
||||
|
|
@ -1922,7 +1924,8 @@ class FastLlamaModel:
|
|||
if old_hf_transfer != "0": os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
||||
|
||||
model_patcher.pre_patch()
|
||||
get_statistics() # For debugging - we use a download counter to see if environments are not breaking
|
||||
# For debugging - we use a download counter to see if environments are not breaking or if HF is down
|
||||
get_statistics(kwargs.get("local_files_only", False))
|
||||
|
||||
if dtype is None:
|
||||
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
|
||||
|
|
|
|||
|
|
@ -210,10 +210,14 @@ class FastLanguageModel(FastLlamaModel):
|
|||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
# Check if pre-quantized models are allowed
|
||||
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.removesuffix("-bnb-4bit")
|
||||
pass
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.lower().removesuffix("-bnb-4bit")
|
||||
# Change -BF16 to all False for 4bit, 8bit etc
|
||||
if model_name.lower().endswith("-bf16"):
|
||||
load_in_4bit = False
|
||||
load_in_8bit = False
|
||||
load_in_16bit = True
|
||||
|
||||
if USE_MODELSCOPE and not os.path.exists(model_name):
|
||||
from modelscope import snapshot_download
|
||||
|
|
@ -327,10 +331,15 @@ class FastLanguageModel(FastLlamaModel):
|
|||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
# Check if pre-quantized models are allowed
|
||||
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.removesuffix("-bnb-4bit")
|
||||
pass
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.lower().removesuffix("-bnb-4bit")
|
||||
# Change -BF16 to all False for 4bit, 8bit etc
|
||||
if model_name.lower().endswith("-bf16"):
|
||||
load_in_4bit = False
|
||||
load_in_8bit = False
|
||||
load_in_16bit = True
|
||||
|
||||
model_config = AutoConfig.from_pretrained(
|
||||
model_name,
|
||||
token = token,
|
||||
|
|
@ -649,10 +658,14 @@ class FastModel(FastBaseModel):
|
|||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
# Check if pre-quantized models are allowed
|
||||
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.removesuffix("-bnb-4bit")
|
||||
pass
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.lower().removesuffix("-bnb-4bit")
|
||||
# Change -BF16 to all False for 4bit, 8bit etc
|
||||
if model_name.lower().endswith("-bf16"):
|
||||
load_in_4bit = False
|
||||
load_in_8bit = False
|
||||
load_in_16bit = True
|
||||
|
||||
# Check modelscope
|
||||
if USE_MODELSCOPE and not os.path.exists(model_name):
|
||||
|
|
@ -870,10 +883,15 @@ class FastModel(FastBaseModel):
|
|||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
# Check if pre-quantized models are allowed
|
||||
# For eg AMD GPUs need blocksize = 128, but our pre-quants are blocksize = 64
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.removesuffix("-bnb-4bit")
|
||||
pass
|
||||
if not ALLOW_PREQUANTIZED_MODELS and model_name.lower().endswith(("-unsloth-bnb-4bit", "-bnb-4bit")):
|
||||
model_name = model_name.lower().removesuffix("-unsloth-bnb-4bit")
|
||||
model_name = model_name.lower().removesuffix("-bnb-4bit")
|
||||
# Change -BF16 to all False for 4bit, 8bit etc
|
||||
if model_name.lower().endswith("-bf16"):
|
||||
load_in_4bit = False
|
||||
load_in_8bit = False
|
||||
load_in_16bit = True
|
||||
|
||||
model_config = AutoConfig.from_pretrained(
|
||||
model_name,
|
||||
token = token,
|
||||
|
|
|
|||
|
|
@ -416,7 +416,8 @@ class FastBaseModel:
|
|||
pass
|
||||
if old_hf_transfer != "0": os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
||||
|
||||
get_statistics() # For debugging - we use a download counter to see if environments are not breaking
|
||||
# For debugging - we use a download counter to see if environments are not breaking or if HF is down
|
||||
get_statistics(kwargs.get("local_files_only", False))
|
||||
|
||||
if dtype is None:
|
||||
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
|
||||
|
|
|
|||
|
|
@ -2565,10 +2565,10 @@ def unsloth_save_pretrained_torchao(
|
|||
"""
|
||||
# first merge the lora weights
|
||||
arguments = dict(locals())
|
||||
arguments["model"] = self
|
||||
arguments["tokenizer"] = tokenizer
|
||||
arguments["push_to_hub"] = False # We save ourselves
|
||||
arguments["save_method"] = "merged_16bit" # Must be 16bit
|
||||
arguments["model"] = self
|
||||
arguments["tokenizer"] = tokenizer
|
||||
arguments["push_to_hub"] = False # We save ourselves
|
||||
arguments["save_method"] = "merged_16bit" # Must be 16bit
|
||||
del arguments["self"]
|
||||
del arguments["torchao_config"]
|
||||
|
||||
|
|
@ -2722,7 +2722,7 @@ def patch_saving_functions(model, vision = False):
|
|||
model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, model)
|
||||
model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model)
|
||||
model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model)
|
||||
model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)
|
||||
model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)
|
||||
model.push_to_hub_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_push_to_hub, model)
|
||||
model.save_pretrained_ggml = types.MethodType(unsloth_convert_lora_to_ggml_and_save_locally, model)
|
||||
pass
|
||||
|
|
@ -2732,7 +2732,7 @@ def patch_saving_functions(model, vision = False):
|
|||
model.save_pretrained_merged = types.MethodType(unsloth_generic_save_pretrained_merged, model)
|
||||
model.push_to_hub_gguf = types.MethodType(unsloth_push_to_hub_gguf, model)
|
||||
model.save_pretrained_gguf = types.MethodType(unsloth_save_pretrained_gguf, model)
|
||||
model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)
|
||||
model.save_pretrained_torchao = types.MethodType(unsloth_save_pretrained_torchao, model)
|
||||
pass
|
||||
return model
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue