From d95f5782d0750c0dfea0bbc512db150f13dc038c Mon Sep 17 00:00:00 2001 From: Nino Risteski <95188570+NinoRisteski@users.noreply.github.com> Date: Thu, 20 Feb 2025 08:32:25 +0100 Subject: [PATCH 1/3] fix an import error (#1767) * fix an import error * Delete .gitignore * Update loader.py * Update save.py --------- Co-authored-by: Daniel Han --- unsloth/models/loader.py | 10 +++++++--- unsloth/save.py | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 39b367e275..186545cf0c 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -24,10 +24,14 @@ from peft import PeftConfig, PeftModel from .loader_utils import get_model_name import os, contextlib, sys try: - from huggingface_hub.utils import get_token + from huggingface_hub import get_token except: - # Old HF Hub versions <= 0.0.25 - from huggingface_hub.utils._token import get_token + try: + from huggingface_hub.utils import get_token + except: + # For older versions of huggingface_hub + from huggingface_hub.utils._token import get_token + pass pass from huggingface_hub import HfFileSystem import importlib.util diff --git a/unsloth/save.py b/unsloth/save.py index 0f75ecfd05..eaddfa05c5 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -31,10 +31,14 @@ from transformers.models.llama.modeling_llama import logger from .tokenizer_utils import fix_sentencepiece_gguf from huggingface_hub import HfApi try: - from huggingface_hub.utils import get_token + from huggingface_hub import get_token except: - # Old HF Hub versions <= 0.0.25 - from huggingface_hub.utils._token import get_token + try: + from huggingface_hub.utils import get_token + except: + # For older versions of huggingface_hub + from huggingface_hub.utils._token import get_token + pass pass from pathlib import Path From 8e5aeacbeec002f5763136a1abfa3634b33dadd6 Mon Sep 17 00:00:00 2001 From: Edd <68678137+Erland366@users.noreply.github.com> Date: Thu, 20 Feb 2025 11:38:48 +0400 Subject: [PATCH 2/3] Convert mask to float (#1762) --- unsloth/models/llama.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 27651be97a..909dfc339b 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -775,9 +775,12 @@ def LlamaModel_fast_forward( self.SWA_mask = True self.GA_mask = False elif attention_mask is not None: - # Fixes https://github.com/unslothai/unsloth/issues/853 # Unsloth needs a 2D mask, not a [2, 1, n, n] mask! + + # https://github.com/pytorch/pytorch/issues/103749 + # Need to convert to float and not using bool + attention_mask = (1.0 - attention_mask.float()) * torch.finfo(inputs_embeds.dtype).min dynamic_SWA_mask = _prepare_4d_causal_attention_mask_for_sdpa( attention_mask, (batch_size, seq_length), From 5b816e735c37328cc4342fbce5ec304dfb573da0 Mon Sep 17 00:00:00 2001 From: Ben <6579034+versipellis@users.noreply.github.com> Date: Wed, 19 Feb 2025 23:40:07 -0800 Subject: [PATCH 3/3] [Windows Support] Add latest `xformers` wheels to pyproject.toml (#1753) * Add latest xformers * Add a couple of lines to docs --- README.md | 7 +++++-- pyproject.toml | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45312a43d1..4bdd7e2893 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ print(f'pip install --upgrade pip && pip install "unsloth[{x}] @ git+https://git ### Windows Installation To run Unsloth directly on Windows: -- Install Triton from this Windows fork and follow the instructions: https://github.com/woct0rdho/triton-windows +- Install Triton from this Windows fork and follow the instructions: https://github.com/woct0rdho/triton-windows (be aware that the Windows fork requires PyTorch >= 2.4 and CUDA 12) - In the SFTTrainer, set `dataset_num_proc=1` to avoid a crashing issue: ```python trainer = SFTTrainer( @@ -202,12 +202,15 @@ trainer = SFTTrainer( ) ``` +### Advanced/Troubleshooting + For **advanced installation instructions** or if you see weird errors during installations: 1. Install `torch` and `triton`. Go to https://pytorch.org to install it. For example `pip install torch torchvision torchaudio triton` 2. Confirm if CUDA is installated correctly. Try `nvcc`. If that fails, you need to install `cudatoolkit` or CUDA drivers. 3. Install `xformers` manually. You can try installing `vllm` and seeing if `vllm` succeeds. Check if `xformers` succeeded with `python -m xformers.info` Go to https://github.com/facebookresearch/xformers. Another option is to install `flash-attn` for Ampere GPUs. -4. Finally, install `bitsandbytes` and check it with `python -m bitsandbytes` +4. Double check that your versions of Python, CUDA, CUDNN, `torch`, `triton`, and `xformers` are compatible with one another. The [PyTorch Compatibility Matrix](https://github.com/pytorch/pytorch/blob/main/RELEASE.md#release-compatibility-matrix) may be useful. +5. Finally, install `bitsandbytes` and check it with `python -m bitsandbytes` ## 📜 [Documentation](https://docs.unsloth.ai) - Go to our official [Documentation](https://docs.unsloth.ai) for saving to GGUF, checkpointing, evaluation and more! diff --git a/pyproject.toml b/pyproject.toml index 59a7c44737..07085adcca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,6 +196,10 @@ cu126onlytorch260 = [ "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-manylinux_2_28_x86_64.whl ; python_version=='3.10' and platform_system == 'Linux'", "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-manylinux_2_28_x86_64.whl ; python_version=='3.11' and platform_system == 'Linux'", "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-manylinux_2_28_x86_64.whl ; python_version=='3.12' and platform_system == 'Linux'", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp39-cp39-win_amd64.whl ; python_version=='3.9' and platform_system == 'Windows'", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp310-cp310-win_amd64.whl ; python_version=='3.10' and platform_system == 'Windows'", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp311-cp311-win_amd64.whl ; python_version=='3.11' and platform_system == 'Windows'", + "xformers @ https://download.pytorch.org/whl/cu126/xformers-0.0.29.post3-cp312-cp312-win_amd64.whl ; python_version=='3.12' and platform_system == 'Windows'", ] cu118 = [ "unsloth[huggingface]",