Merge main into grpo-fix-on-pr3754
This commit is contained in:
danielhanchen
commit
3240ab3391
23 changed files with 603 additions and 182 deletions
2
.github/FUNDING.yml
vendored
2
.github/FUNDING.yml
vendored
|
|
@ -3,7 +3,7 @@
|
|||
github: unslothai
|
||||
patreon: # Replace with a single Patreon username
|
||||
open_collective: # Replace with a single Open Collective username
|
||||
ko_fi: unsloth
|
||||
ko_fi: # unsloth
|
||||
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||
liberapay: # Replace with a single Liberapay username
|
||||
|
|
|
|||
126
README.md
126
README.md
|
|
@ -1,6 +1,6 @@
|
|||
<div align="center">
|
||||
|
||||
<a href="https://docs.unsloth.ai"><picture>
|
||||
<a href="https://unsloth.ai/docs"><picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20white%20text.png">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20black%20text.png">
|
||||
<img alt="unsloth logo" src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20black%20text.png" height="110" style="max-width: 100%;">
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
<a href="https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-Fine-tuning.ipynb"><img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/start free finetune button.png" width="154"></a>
|
||||
<a href="https://discord.com/invite/unsloth"><img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/Discord button.png" width="165"></a>
|
||||
<a href="https://docs.unsloth.ai"><img src="https://raw.githubusercontent.com/unslothai/unsloth/refs/heads/main/images/Documentation%20Button.png" width="137"></a>
|
||||
<a href="https://unsloth.ai/docs"><img src="https://raw.githubusercontent.com/unslothai/unsloth/refs/heads/main/images/Documentation%20Button.png" width="137"></a>
|
||||
|
||||
### Train gpt-oss, DeepSeek, Gemma, Qwen & Llama 2x faster with 70% less VRAM!
|
||||
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
## ✨ Train for Free
|
||||
|
||||
Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then export your trained model to GGUF, llama.cpp, Ollama, vLLM, SGLang or Hugging Face.
|
||||
Notebooks are beginner friendly. Read our [guide](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide). Add dataset, run, then deploy your trained model.
|
||||
|
||||
| Model | Free Notebooks | Performance | Memory use |
|
||||
|-----------|---------|--------|----------|
|
||||
|
|
@ -34,9 +34,9 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st
|
|||
| **Llama 3.2 Conversational** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(1B_and_3B)-Conversational.ipynb) | 2x faster | 70% less |
|
||||
| **Orpheus-TTS (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Orpheus_(3B)-TTS.ipynb) | 1.5x faster | 50% less |
|
||||
|
||||
- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://docs.unsloth.ai/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), **[TTS](https://docs.unsloth.ai/get-started/unsloth-notebooks#text-to-speech-tts-notebooks)** & [Vision](https://docs.unsloth.ai/get-started/unsloth-notebooks#vision-multimodal-notebooks)
|
||||
- See [all our models](https://docs.unsloth.ai/get-started/all-our-models) and [all our notebooks](https://docs.unsloth.ai/get-started/unsloth-notebooks)
|
||||
- See detailed documentation for Unsloth [here](https://docs.unsloth.ai/)
|
||||
- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://unsloth.ai/docs/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), [TTS](https://unsloth.ai/docs/get-started/unsloth-notebooks#text-to-speech-tts-notebooks) & [Vision](https://unsloth.ai/docs/get-started/unsloth-notebooks#vision-multimodal-notebooks)
|
||||
- See [all our models](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [all our notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks)
|
||||
- See detailed documentation for Unsloth [here](https://unsloth.ai/docs)
|
||||
|
||||
## ⚡ Quickstart
|
||||
### Linux or WSL
|
||||
|
|
@ -44,33 +44,35 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st
|
|||
pip install unsloth
|
||||
```
|
||||
### Windows
|
||||
For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://docs.unsloth.ai/get-started/installing-+-updating/windows-installation).
|
||||
For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://unsloth.ai/docs/get-started/install-and-update/windows-installation).
|
||||
|
||||
### Docker
|
||||
Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://docs.unsloth.ai/get-started/install-and-update/docker).
|
||||
Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://unsloth.ai/docs/get-started/install-and-update/docker).
|
||||
|
||||
### Blackwell & DGX Spark
|
||||
For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://docs.unsloth.ai/basics/training-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://docs.unsloth.ai/new/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details.
|
||||
For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details.
|
||||
|
||||
## 🦥 Unsloth News
|
||||
- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://docs.unsloth.ai/new/3x-faster-training-packing)
|
||||
- **Ministral 3** by Mistral: Run Ministral 3 or fine-tune with vision/RL sodoku notebooks. [Guide](https://docs.unsloth.ai/new/ministral-3) • [Notebooks](https://docs.unsloth.ai/new/ministral-3#fine-tuningb)
|
||||
- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://docs.unsloth.ai/new/500k-context-length-fine-tuning)
|
||||
- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://docs.unsloth.ai/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://docs.unsloth.ai/new/deepseek-ocr-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb)
|
||||
- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://docs.unsloth.ai/new/how-to-train-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth)
|
||||
- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://docs.unsloth.ai/new/gpt-oss-reinforcement-learning)
|
||||
- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://docs.unsloth.ai/new/vision-reinforcement-learning-vlm-rl)
|
||||
- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://docs.unsloth.ai/new/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://docs.unsloth.ai/basics/gpt-oss). 20B works on 14GB VRAM. 120B on 65GB.
|
||||
- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://unsloth.ai/docs/new/3x-faster-training-packing)
|
||||
- **New Mistral**: Run Ministral 3 or Devstral 2 and fine-tune with vision/RL sodoku notebooks. [Guide](https://unsloth.ai/docs/models/ministral-3) • [Notebooks](https://unsloth.ai/docs/models/ministral-3#fine-tuning-ministral-3)
|
||||
- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://unsloth.ai/docs/new/500k-context-length-fine-tuning)
|
||||
- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://unsloth.ai/docs/models/deepseek-ocr-how-to-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb)
|
||||
- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://unsloth.ai/docs/new/how-to-fine-tune-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth)
|
||||
- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/gpt-oss-reinforcement-learning)
|
||||
- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/vision-reinforcement-learning-vlm-rl)
|
||||
- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune). 20B works on 14GB VRAM. 120B on 65GB.
|
||||
|
||||
<details>
|
||||
<summary>Click for more news</summary>
|
||||
|
||||
- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://docs.unsloth.ai/new/quantization-aware-training-qat)
|
||||
- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://docs.unsloth.ai/new/memory-efficient-rl)
|
||||
- **Gemma 3n** by Google: [Read Blog](https://docs.unsloth.ai/basics/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339).
|
||||
- **[Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`.
|
||||
- **[Qwen3](https://docs.unsloth.ai/basics/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM.
|
||||
- Introducing **[Dynamic 2.0](https://docs.unsloth.ai/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot.
|
||||
- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://docs.unsloth.ai/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`.
|
||||
- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://unsloth.ai/docs/basics/quantization-aware-training-qat)
|
||||
- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/memory-efficient-rl)
|
||||
- **Gemma 3n** by Google: [Read Blog](https://unsloth.ai/docs/models/gemma-3-how-to-run-and-fine-tune/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339).
|
||||
- **[Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`.
|
||||
- **[Qwen3](https://unsloth.ai/docs/models/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM.
|
||||
- Introducing **[Dynamic 2.0](https://unsloth.ai/docs/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot.
|
||||
- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://unsloth.ai/docs/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`.
|
||||
- 📣 [DeepSeek-R1](https://unsloth.ai/blog/deepseek-r1) - run or fine-tune them [with our guide](https://unsloth.ai/blog/deepseek-r1). All model uploads: [here](https://huggingface.co/collections/unsloth/deepseek-r1-all-versions-678e1c48f5d2fce87892ace5).
|
||||
- 📣 Introducing Long-context [Reasoning (GRPO)](https://unsloth.ai/blog/grpo) in Unsloth. Train your own reasoning model with just 5GB VRAM. Transform Llama, Phi, Mistral etc. into reasoning LLMs!
|
||||
- 📣 Introducing Unsloth [Dynamic 4-bit Quantization](https://unsloth.ai/blog/dynamic-4bit)! We dynamically opt not to quantize certain parameters and this greatly increases accuracy while only using <10% more VRAM than BnB 4-bit. See our collection on [Hugging Face here.](https://huggingface.co/collections/unsloth/unsloth-4-bit-dynamic-quants-67503bb873f89e15276c44e7)
|
||||
|
|
@ -84,27 +86,29 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](
|
|||
</details>
|
||||
|
||||
## 🔗 Links and Resources
|
||||
| Type | Links |
|
||||
| ------------------------------- | --------------------------------------- |
|
||||
| <img width="15" src="https://redditinc.com/hs-fs/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png" /> **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth)|
|
||||
| 📚 **Documentation & Wiki** | [Read Our Docs](https://docs.unsloth.ai) |
|
||||
| <img width="16" src="https://upload.wikimedia.org/wikipedia/commons/6/6f/Logo_of_Twitter.svg" /> **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai)|
|
||||
| 💾 **Installation** | [Pip & Docker Install](https://docs.unsloth.ai/get-started/installing-+-updating)|
|
||||
| 🔮 **Our Models** | [Unsloth Catalog](https://docs.unsloth.ai/get-started/all-our-models)|
|
||||
| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog)|
|
||||
| Type | Links |
|
||||
| ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ |
|
||||
| <img width="15" src="https://redditinc.com/hs-fs/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png" /> **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth) |
|
||||
| 📚 **Documentation & Wiki** | [Read Our Docs](https://unsloth.ai/docs) |
|
||||
| <img width="13" src="https://upload.wikimedia.org/wikipedia/commons/0/09/X_(formerly_Twitter)_logo_late_2025.svg" /> **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai) |
|
||||
| 💾 **Installation** | [Pip & Docker Install](https://unsloth.ai/docs/get-started/install-and-update) |
|
||||
| 🔮 **Our Models** | [Unsloth Catalog](https://unsloth.ai/docs/get-started/unsloth-model-catalog) |
|
||||
| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog) |
|
||||
|
||||
## ⭐ Key Features
|
||||
- Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training
|
||||
- Supports **all models** including [TTS](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://docs.unsloth.ai/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth.
|
||||
- The most efficient library for [Reinforcement Learning (RL)](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc.
|
||||
- **0% loss in accuracy** - no approximation methods - all exact.
|
||||
- Supports NVIDIA (since 2018), [AMD](https://docs.unsloth.ai/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc)
|
||||
- Works on **Linux**, WSL and **Windows**
|
||||
- All kernels written in [OpenAI's Triton](https://openai.com/index/triton/) language. Manual backprop engine.
|
||||
- If you trained a model with 🦥Unsloth, you can use this cool sticker! <img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/made with unsloth.png" width="200" align="center" />
|
||||
|
||||
* Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training
|
||||
* Supports **all models** including [TTS](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://unsloth.ai/docs/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth.
|
||||
* The most efficient library for [Reinforcement Learning (RL)](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc.
|
||||
* **0% loss in accuracy** - no approximation methods - all exact.
|
||||
* Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face.
|
||||
* Supports NVIDIA (since 2018), [AMD](https://unsloth.ai/docs/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc)
|
||||
* Works on **Linux**, WSL and **Windows**
|
||||
* All kernels written in OpenAI's Triton language. Manual backprop engine.
|
||||
* If you trained a model with 🦥Unsloth, you can use this cool sticker! <img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/made with unsloth.png" width="200" align="center" />
|
||||
|
||||
## 💾 Install Unsloth
|
||||
You can also see our docs for more detailed installation and updating instructions [here](https://docs.unsloth.ai/get-started/installing-+-updating).
|
||||
You can also see our docs for more detailed installation and updating instructions [here](https://unsloth.ai/docs/get-started/install-and-update).
|
||||
|
||||
Unsloth supports Python 3.13 or lower.
|
||||
|
||||
|
|
@ -124,7 +128,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions.
|
|||
You should install the latest driver for your GPU. Download drivers here: [NVIDIA GPU Driver](https://www.nvidia.com/Download/index.aspx).
|
||||
|
||||
3. **Install Visual Studio C++:**
|
||||
You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://docs.unsloth.ai/get-started/installing-+-updating).
|
||||
You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://unsloth.ai/docs/get-started/install-and-update/windows-installation#method-3-windows-directly).
|
||||
|
||||
5. **Install CUDA Toolkit:**
|
||||
Follow the instructions to install [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit-archive).
|
||||
|
|
@ -139,19 +143,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions.
|
|||
pip install unsloth
|
||||
```
|
||||
|
||||
#### Notes
|
||||
To run Unsloth directly on Windows:
|
||||
- Install Triton from this Windows fork and follow the instructions [here](https://github.com/woct0rdho/triton-windows) (be aware that the Windows fork requires PyTorch >= 2.4 and CUDA 12)
|
||||
- In the `SFTConfig`, set `dataset_num_proc=1` to avoid a crashing issue:
|
||||
```python
|
||||
SFTConfig(
|
||||
dataset_num_proc=1,
|
||||
...
|
||||
)
|
||||
```
|
||||
|
||||
#### Advanced/Troubleshooting
|
||||
|
||||
For **advanced installation instructions** or if you see weird errors during installations:
|
||||
|
||||
First try using an isolated environment via then `pip install unsloth`
|
||||
|
|
@ -268,7 +260,7 @@ print(f'pip install --upgrade pip && pip install --no-deps git+https://github.co
|
|||
```
|
||||
### Docker Installation
|
||||
You can use our pre-built Docker container with all dependencies to use Unsloth instantly with no setup required.
|
||||
[Read our guide](https://docs.unsloth.ai/get-started/install-and-update/docker).
|
||||
[Read our guide](https://unsloth.ai/docs/get-started/install-and-update/docker).
|
||||
|
||||
This container requires installing [NVIDIA's Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
|
||||
|
||||
|
|
@ -283,9 +275,9 @@ docker run -d -e JUPYTER_PASSWORD="mypassword" \
|
|||
Access Jupyter Lab at `http://localhost:8888` and start fine-tuning!
|
||||
|
||||
## 📜 Documentation
|
||||
- Go to our official [Documentation](https://docs.unsloth.ai) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more!
|
||||
- Read our Guides for: [Fine-tuning](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), [Vision](https://docs.unsloth.ai/basics/vision-fine-tuning) and [any model](https://docs.unsloth.ai/models/tutorials-how-to-fine-tune-and-run-llms).
|
||||
- We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code.
|
||||
* Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://unsloth.ai/docs/basics/inference-and-deployment), [saving to GGUF](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf), [checkpointing](https://unsloth.ai/docs/basics/finetuning-from-last-checkpoint), [evaluation](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide#evaluation) and more!
|
||||
* Read our Guides for: [Fine-tuning](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), [Vision](https://unsloth.ai/docs/basics/vision-fine-tuning) and [any model](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms).
|
||||
* We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code.
|
||||
|
||||
Unsloth example code to fine-tune gpt-oss-20b:
|
||||
|
||||
|
|
@ -310,8 +302,9 @@ model, tokenizer = FastModel.from_pretrained(
|
|||
max_seq_length = 2048, # Choose any for long context!
|
||||
load_in_4bit = True, # 4-bit quantization. False = 16-bit LoRA.
|
||||
load_in_8bit = False, # 8-bit quantization
|
||||
load_in_16bit = False, # [NEW!] 16-bit LoRA
|
||||
load_in_16bit = False, # 16-bit LoRA
|
||||
full_finetuning = False, # Use for full fine-tuning.
|
||||
trust_remote_code = False, # Enable to support new models
|
||||
# token = "hf_...", # use one if using gated models
|
||||
)
|
||||
|
||||
|
|
@ -350,7 +343,7 @@ trainer = SFTTrainer(
|
|||
)
|
||||
trainer.train()
|
||||
|
||||
# Go to https://docs.unsloth.ai for advanced tips like
|
||||
# Go to https://unsloth.ai/docs for advanced tips like
|
||||
# (1) Saving to GGUF / merging to 16bit for vLLM or SGLang
|
||||
# (2) Continued training from a saved LoRA adapter
|
||||
# (3) Adding an evaluation loop / OOMs
|
||||
|
|
@ -359,14 +352,15 @@ trainer.train()
|
|||
|
||||
<a name="RL"></a>
|
||||
## 💡 Reinforcement Learning
|
||||
[RL](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) including [GRPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), **FP8** traning, DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth.
|
||||
Read our [Reinforcement Learning Guide](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters.
|
||||
[RL](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) including [GRPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), [**FP8** training](https://unsloth.ai/docs/new/fp8-reinforcement-learning), DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth.
|
||||
|
||||
Read our [Reinforcement Learning Guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters.
|
||||
|
||||
List of RL notebooks:
|
||||
- gpt-oss GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-GRPO.ipynb)
|
||||
- Qwen2.5-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen2_5_7B_VL_GRPO.ipynb)
|
||||
- - ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- Qwen2.3-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_VL_(8B)-Vision-GRPO.ipynb)
|
||||
- Advanced Qwen3 GRPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb)
|
||||
- ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- ORPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-ORPO.ipynb)
|
||||
- DPO Zephyr notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Zephyr_(7B)-DPO.ipynb)
|
||||
- KTO notebook: [Link](https://colab.research.google.com/drive/1MRgGtLWuZX4ypSfGguFgC-IblTvO2ivM?usp=sharing)
|
||||
|
|
@ -426,6 +420,4 @@ You can cite the Unsloth repo as follows:
|
|||
- The [llama.cpp library](https://github.com/ggml-org/llama.cpp) that lets users save models with Unsloth
|
||||
- The Hugging Face team and their libraries: [transformers](https://github.com/huggingface/transformers) and [TRL](https://github.com/huggingface/trl)
|
||||
- The Pytorch and [Torch AO](https://github.com/unslothai/unsloth/pull/3391) team for their contributions
|
||||
- [Erik](https://github.com/erikwijmans) for his help adding [Apple's ML Cross Entropy](https://github.com/apple/ml-cross-entropy) in Unsloth
|
||||
- [Etherl](https://github.com/Etherll) for adding support for [TTS, diffusion and BERT models](https://github.com/unslothai/notebooks/pull/34)
|
||||
- And of course for every single person who has contributed or has used Unsloth!
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ huggingfacenotorch = [
|
|||
]
|
||||
huggingface = [
|
||||
"unsloth[huggingfacenotorch]",
|
||||
"unsloth_zoo>=2025.12.7",
|
||||
"unsloth_zoo>=2026.1.1",
|
||||
"torchvision",
|
||||
"unsloth[triton]",
|
||||
]
|
||||
|
|
@ -523,7 +523,7 @@ colab-ampere-torch220 = [
|
|||
"flash-attn>=2.6.3 ; ('linux' in sys_platform)",
|
||||
]
|
||||
colab-new = [
|
||||
"unsloth_zoo>=2025.12.7",
|
||||
"unsloth_zoo>=2026.1.1",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,!=4.57.0,<=4.57.3",
|
||||
|
|
|
|||
|
|
@ -30,16 +30,19 @@ from .import_fixes import (
|
|||
check_fbgemm_gpu_version,
|
||||
torchvision_compatibility_check,
|
||||
fix_diffusers_warnings,
|
||||
fix_huggingface_hub,
|
||||
)
|
||||
|
||||
fix_message_factory_issue()
|
||||
check_fbgemm_gpu_version()
|
||||
torchvision_compatibility_check()
|
||||
fix_diffusers_warnings()
|
||||
fix_huggingface_hub()
|
||||
del fix_message_factory_issue
|
||||
del check_fbgemm_gpu_version
|
||||
del torchvision_compatibility_check
|
||||
del fix_diffusers_warnings
|
||||
del fix_huggingface_hub
|
||||
|
||||
# This check is critical because Unsloth optimizes these libraries by modifying
|
||||
# their code at import time. If they're imported first, the original (slower,
|
||||
|
|
@ -76,7 +79,7 @@ from importlib.metadata import PackageNotFoundError
|
|||
# Check for unsloth_zoo
|
||||
try:
|
||||
unsloth_zoo_version = importlib_version("unsloth_zoo")
|
||||
if Version(unsloth_zoo_version) < Version("2025.12.4"):
|
||||
if Version(unsloth_zoo_version) < Version("2026.1.1"):
|
||||
print(
|
||||
"Unsloth: Please update Unsloth and Unsloth-Zoo to the latest version!\n"
|
||||
"Do this via `pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo`"
|
||||
|
|
@ -123,6 +126,7 @@ from .import_fixes import (
|
|||
fix_xformers_performance_issue,
|
||||
fix_vllm_aimv2_issue,
|
||||
fix_vllm_guided_decoding_params,
|
||||
fix_vllm_pdl_blackwell,
|
||||
ignore_logger_messages,
|
||||
patch_ipykernel_hf_xet,
|
||||
patch_trackio,
|
||||
|
|
@ -135,6 +139,7 @@ from .import_fixes import (
|
|||
fix_xformers_performance_issue()
|
||||
fix_vllm_aimv2_issue()
|
||||
fix_vllm_guided_decoding_params()
|
||||
fix_vllm_pdl_blackwell()
|
||||
ignore_logger_messages()
|
||||
patch_ipykernel_hf_xet()
|
||||
patch_trackio()
|
||||
|
|
@ -146,6 +151,7 @@ fix_executorch()
|
|||
del fix_xformers_performance_issue
|
||||
del fix_vllm_aimv2_issue
|
||||
del fix_vllm_guided_decoding_params
|
||||
del fix_vllm_pdl_blackwell
|
||||
del ignore_logger_messages
|
||||
del patch_ipykernel_hf_xet
|
||||
del patch_trackio
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ __all__ = [
|
|||
|
||||
import torch
|
||||
import functools
|
||||
import inspect
|
||||
from unsloth_zoo.utils import Version
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from packaging.version import Version as TrueVersion
|
|||
import re
|
||||
import logging
|
||||
import textwrap
|
||||
import warnings
|
||||
|
||||
# We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults.
|
||||
UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in (
|
||||
|
|
@ -97,6 +98,12 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1":
|
|||
sys.stderr = HidePrintMessage(sys.stderr)
|
||||
# https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52
|
||||
sys.stderr.add_filter("TMA benchmarks will be running")
|
||||
# Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0
|
||||
logging.getLogger("torchao").setLevel(logging.ERROR)
|
||||
# SyntaxWarning: invalid escape sequence '\.'
|
||||
warnings.filterwarnings(
|
||||
"ignore", message = "invalid escape sequence", category = SyntaxWarning
|
||||
)
|
||||
|
||||
|
||||
# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'
|
||||
|
|
@ -539,3 +546,128 @@ def fix_executorch():
|
|||
def fix_diffusers_warnings():
|
||||
# Silence Flax classes are deprecated and will be removed in Diffusers v1.0.0.
|
||||
os.environ["DIFFUSERS_VERBOSITY"] = "error"
|
||||
|
||||
|
||||
def fix_huggingface_hub():
|
||||
# huggingface_hub.is_offline_mode got removed, so add it back
|
||||
import huggingface_hub
|
||||
|
||||
if not hasattr(huggingface_hub, "is_offline_mode"):
|
||||
huggingface_hub.is_offline_mode = (
|
||||
lambda: huggingface_hub.constants.HF_HUB_OFFLINE
|
||||
)
|
||||
|
||||
|
||||
def fix_vllm_pdl_blackwell():
|
||||
"""
|
||||
Fix vLLM PDL (Programmatic Dependent Launch) bug on Blackwell GPUs (SM100).
|
||||
|
||||
The issue: vLLM's LoRA Triton kernels use tl.extra.cuda.gdc_wait() for PDL
|
||||
optimization on SM90+ GPUs. This fails on SM100 (B200/B100) during CUDA graph
|
||||
capture because Triton's pipeliner can't handle gdc_wait in complex kernels.
|
||||
|
||||
See: https://github.com/vllm-project/vllm/issues/30872
|
||||
"""
|
||||
if importlib.util.find_spec("vllm") is None:
|
||||
return
|
||||
|
||||
# Check if any CUDA GPU is SM100 (Blackwell)
|
||||
try:
|
||||
import torch
|
||||
|
||||
if not torch.cuda.is_available():
|
||||
return
|
||||
|
||||
# Scan all GPUs for SM100 - fix applies globally via env var and monkey-patch
|
||||
has_sm100 = False
|
||||
sm100_gpu_name = None
|
||||
for i in range(torch.cuda.device_count()):
|
||||
major, minor = torch.cuda.get_device_capability(i)
|
||||
if major == 10:
|
||||
has_sm100 = True
|
||||
sm100_gpu_name = torch.cuda.get_device_name(i)
|
||||
break
|
||||
|
||||
if not has_sm100:
|
||||
return
|
||||
except Exception:
|
||||
return
|
||||
|
||||
# Helper to check if module spec exists
|
||||
def _spec_exists(name):
|
||||
try:
|
||||
return importlib.util.find_spec(name) is not None
|
||||
except (ModuleNotFoundError, ValueError):
|
||||
return False
|
||||
|
||||
# Check if vLLM has the PDL-related modules before doing internet check
|
||||
has_utils = _spec_exists("vllm.lora.ops.triton_ops.utils")
|
||||
has_expand_op = _spec_exists("vllm.lora.ops.triton_ops.lora_expand_op")
|
||||
has_shrink_op = _spec_exists("vllm.lora.ops.triton_ops.lora_shrink_op")
|
||||
|
||||
if not has_utils and not has_expand_op and not has_shrink_op:
|
||||
# Old vLLM version without PDL support - nothing to patch
|
||||
return
|
||||
|
||||
# Check if vLLM version includes the fix
|
||||
VLLM_PDL_FIX_VERSION = "0.13.2"
|
||||
try:
|
||||
vllm_version = Version(importlib_version("vllm"))
|
||||
if vllm_version > Version(VLLM_PDL_FIX_VERSION):
|
||||
logger.info(
|
||||
f"Unsloth: SM100 ({sm100_gpu_name}) detected but vLLM {vllm_version} "
|
||||
f"should include PDL fix - skipping workaround"
|
||||
)
|
||||
return
|
||||
except Exception as e:
|
||||
logger.debug(
|
||||
f"Unsloth: vLLM version check failed ({e}), applying PDL workaround."
|
||||
)
|
||||
|
||||
# Apply the PDL fix
|
||||
os.environ["TRITON_DISABLE_PDL"] = "1"
|
||||
|
||||
def fake_supports_pdl(*args, **kwargs):
|
||||
return False
|
||||
|
||||
patched = []
|
||||
|
||||
# First, patch the source module (utils.py) where supports_pdl is defined.
|
||||
# This is critical because supports_pdl uses @lru_cache - we must clear the
|
||||
# cache to prevent stale cached results from the original function.
|
||||
try:
|
||||
utils_module = importlib.import_module("vllm.lora.ops.triton_ops.utils")
|
||||
if hasattr(utils_module, "supports_pdl"):
|
||||
original_fn = utils_module.supports_pdl
|
||||
if hasattr(original_fn, "cache_clear"):
|
||||
original_fn.cache_clear()
|
||||
utils_module.supports_pdl = fake_supports_pdl
|
||||
patched.append("utils")
|
||||
except (ImportError, ModuleNotFoundError, AttributeError):
|
||||
pass
|
||||
|
||||
# Also patch the consumer modules that import supports_pdl from utils.
|
||||
# This ensures the patched function is used even if the module was already
|
||||
# imported before this fix runs.
|
||||
consumer_modules = {
|
||||
"lora_expand_op": "vllm.lora.ops.triton_ops.lora_expand_op",
|
||||
"lora_shrink_op": "vllm.lora.ops.triton_ops.lora_shrink_op",
|
||||
"fused_moe_lora_op": "vllm.lora.ops.triton_ops.fused_moe_lora_op",
|
||||
}
|
||||
for name, path in consumer_modules.items():
|
||||
try:
|
||||
module = importlib.import_module(path)
|
||||
if hasattr(module, "supports_pdl"):
|
||||
module.supports_pdl = fake_supports_pdl
|
||||
patched.append(name)
|
||||
except (ImportError, ModuleNotFoundError, AttributeError):
|
||||
pass
|
||||
|
||||
if patched:
|
||||
logger.info(
|
||||
f"Unsloth: Applied PDL fix for SM100 ({sm100_gpu_name}) - "
|
||||
f"patched: {', '.join(patched)}"
|
||||
)
|
||||
else:
|
||||
# Just set the env var - vLLM might be an older version without supports_pdl
|
||||
logger.info(f"Unsloth: Set TRITON_DISABLE_PDL=1 for SM100 ({sm100_gpu_name})")
|
||||
|
|
|
|||
|
|
@ -379,9 +379,22 @@ class LoRA_QKV(torch.autograd.Function):
|
|||
):
|
||||
dtype = X.dtype
|
||||
|
||||
Q = matmul_lora(X, QW, QW_quant, QA, QB, QS)
|
||||
K = matmul_lora(X, KW, KW_quant, KA, KB, KS)
|
||||
V = matmul_lora(X, VW, VW_quant, VA, VB, VS)
|
||||
# bitsandbytes 8-bit matmul expects 2D inputs.
|
||||
# TorchInductor/AOTAutograd fails on 3D tensors during backward,
|
||||
# so we explicitly flatten the sequence dimension.
|
||||
orig_shape = X.shape
|
||||
X_for_matmul = X
|
||||
if X.dim() == 3:
|
||||
X_for_matmul = X.view(-1, X.shape[-1])
|
||||
Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)
|
||||
K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)
|
||||
V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)
|
||||
|
||||
# Restore original shape after matmul
|
||||
if len(orig_shape) == 3:
|
||||
Q = Q.view(orig_shape[0], orig_shape[1], -1)
|
||||
K = K.view(orig_shape[0], orig_shape[1], -1)
|
||||
V = V.view(orig_shape[0], orig_shape[1], -1)
|
||||
|
||||
ctx.custom_saved_tensors = (
|
||||
QW,
|
||||
|
|
|
|||
|
|
@ -312,8 +312,8 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
|
|||
_, n_heads_K, _, _ = K.shape
|
||||
|
||||
# Inplace rotary embedding is generally fine
|
||||
Q_out = Q.clone() if not Q.is_contiguous else Q
|
||||
K_out = K.clone() if not K.is_contiguous else K
|
||||
Q_out = Q.clone() if not Q.is_contiguous() else Q
|
||||
K_out = K.clone() if not K.is_contiguous() else K
|
||||
|
||||
if has_indices:
|
||||
# TRL's rotary indices are always in int32, so casting is just for safety
|
||||
|
|
@ -383,21 +383,21 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
|
|||
else ctx.cos.new_empty(1, dtype = torch.int32)
|
||||
)
|
||||
|
||||
# Inplace rotary embedding is generally fine
|
||||
dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ
|
||||
dK_out = dK.clone() if not dK.is_contiguous() else dK
|
||||
|
||||
Q_batch_stride, Q_head_stride, Q_seq_stride = (
|
||||
dQ.stride(0),
|
||||
dQ.stride(1),
|
||||
dQ.stride(2),
|
||||
dQ_out.stride(0),
|
||||
dQ_out.stride(1),
|
||||
dQ_out.stride(2),
|
||||
)
|
||||
K_batch_stride, K_head_stride, K_seq_stride = (
|
||||
dK.stride(0),
|
||||
dK.stride(1),
|
||||
dK.stride(2),
|
||||
dK_out.stride(0),
|
||||
dK_out.stride(1),
|
||||
dK_out.stride(2),
|
||||
)
|
||||
|
||||
# Inplace rotary embedding is generally fine
|
||||
dQ_out = dQ.clone() if not dQ.is_contiguous else dQ
|
||||
dK_out = dK.clone() if not dK.is_contiguous else dK
|
||||
|
||||
with torch_gpu_device(dQ.device):
|
||||
_rope_embedding_QK[(batch * ctx.seq_len, ctx.n_heads_Q)](
|
||||
dQ_out,
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2025.12.9"
|
||||
__version__ = "2026.1.1"
|
||||
|
||||
__all__ = [
|
||||
"SUPPORTS_BFLOAT16",
|
||||
|
|
@ -73,6 +73,7 @@ __all__ = [
|
|||
"verify_fp8_support_if_applicable",
|
||||
"_get_inference_mode_context_manager",
|
||||
"hf_login",
|
||||
"make_fast_generate_wrapper",
|
||||
]
|
||||
|
||||
import torch
|
||||
|
|
@ -1981,9 +1982,10 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m
|
|||
type(init_lora_weights) is bool
|
||||
or init_lora_weights == "gaussian"
|
||||
or init_lora_weights == "loftq"
|
||||
or init_lora_weights == "corda"
|
||||
):
|
||||
raise ValueError(
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].'
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
|
||||
)
|
||||
|
||||
if init_lora_weights == "loftq":
|
||||
|
|
@ -2197,6 +2199,18 @@ def _prepare_model_for_qat(
|
|||
from torchao.quantization.granularity import PerGroup, PerAxis
|
||||
from torchao.quantization.qat import QATConfig
|
||||
|
||||
# Gemma3 models have issues with int8 embedding quantization due to their
|
||||
# large vocabulary size (262144). Auto-switch to int4 weight-only instead.
|
||||
if qat_scheme == "int8-int4":
|
||||
model_types = get_transformers_model_type(model.config)
|
||||
is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types)
|
||||
if is_gemma3:
|
||||
print(
|
||||
"Unsloth: Gemma3 has a large vocabulary causing int8 embedding issues. "
|
||||
"Switching to int4 weight-only QAT for training stability."
|
||||
)
|
||||
qat_scheme = "int4"
|
||||
|
||||
if not isinstance(qat_scheme, TorchAOConfig):
|
||||
torchao_config: Optional[TorchAOConfig] = None
|
||||
if qat_scheme == "fp8-int4":
|
||||
|
|
@ -2365,3 +2379,59 @@ def hf_login(token: Optional[str] = None) -> Optional[str]:
|
|||
except Exception as e:
|
||||
logger.info(f"Failed to login to huggingface using token with error: {e}")
|
||||
return token
|
||||
|
||||
|
||||
def make_fast_generate_wrapper(original_generate):
|
||||
"""
|
||||
Creates a wrapper around model.generate that checks for incorrect
|
||||
vLLM-style usage when fast_inference=False.
|
||||
"""
|
||||
|
||||
@functools.wraps(original_generate)
|
||||
def _fast_generate_wrapper(*args, **kwargs):
|
||||
# Check for vLLM-specific arguments
|
||||
if "sampling_params" in kwargs:
|
||||
raise ValueError(
|
||||
"Unsloth: `sampling_params` is only supported when `fast_inference=True` (vLLM). "
|
||||
"Since `fast_inference=False`, use HuggingFace generate arguments instead:\n"
|
||||
" model.fast_generate(**tokens.to('cuda'), max_new_tokens=64, temperature=1.0, top_p=0.95)"
|
||||
)
|
||||
|
||||
if "lora_request" in kwargs:
|
||||
raise ValueError(
|
||||
"Unsloth: `lora_request` is only supported when `fast_inference=True` (vLLM). "
|
||||
"Since `fast_inference=False`, LoRA weights are already merged into the model."
|
||||
)
|
||||
|
||||
# Check if first positional argument is a string or list of strings
|
||||
if len(args) > 0:
|
||||
first_arg = args[0]
|
||||
is_string_input = False
|
||||
|
||||
if isinstance(first_arg, str):
|
||||
is_string_input = True
|
||||
elif isinstance(first_arg, (list, tuple)) and len(first_arg) > 0:
|
||||
if isinstance(first_arg[0], str):
|
||||
is_string_input = True
|
||||
|
||||
if is_string_input:
|
||||
raise ValueError(
|
||||
"Unsloth: Passing text strings to `fast_generate` is only supported "
|
||||
"when `fast_inference=True` (vLLM). Since `fast_inference=False`, you must "
|
||||
"tokenize the input first:\n\n"
|
||||
" messages = tokenizer.apply_chat_template(\n"
|
||||
' [{"role": "user", "content": "Your prompt here"}],\n'
|
||||
" tokenize=True, add_generation_prompt=True,\n"
|
||||
' return_tensors="pt", return_dict=True\n'
|
||||
" )\n"
|
||||
" output = model.fast_generate(\n"
|
||||
" **messages.to('cuda'),\n"
|
||||
" max_new_tokens=64,\n"
|
||||
" temperature=1.0,\n"
|
||||
" )"
|
||||
)
|
||||
|
||||
# Call original generate
|
||||
return original_generate(*args, **kwargs)
|
||||
|
||||
return _fast_generate_wrapper
|
||||
|
|
|
|||
|
|
@ -344,8 +344,8 @@ def CohereAttention_fast_forward_inference(
|
|||
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
|
||||
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
|
||||
if self.use_qk_norm:
|
||||
Q = fast_layernorm_inference(self.q_norm, Q, self.q_norm_out_weight)
|
||||
K = fast_layernorm_inference(self.k_norm, K, self.k_norm_out_weight)
|
||||
Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight)
|
||||
Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight)
|
||||
|
||||
# cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
|
||||
# Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
|
||||
|
|
@ -479,7 +479,7 @@ def CohereModel_fast_forward_inference(
|
|||
)
|
||||
)
|
||||
|
||||
hidden_states_mlp = fast_swiglu_inference(self.mlp, hidden_states)
|
||||
hidden_states_mlp = fast_swiglu_inference(decoder_layer.mlp, hidden_states)
|
||||
residual += hidden_states_attention
|
||||
residual += hidden_states_mlp
|
||||
hidden_states = residual
|
||||
|
|
|
|||
|
|
@ -456,9 +456,9 @@ def FalconH1DecoderLayer_fast_forward(
|
|||
# Fully Connected
|
||||
residual = hidden_states
|
||||
hidden_states = fast_rms_layernorm_inference(
|
||||
self.post_attention_layernorm, hidden_states
|
||||
self.pre_ff_layernorm, hidden_states
|
||||
)
|
||||
hidden_states = fast_swiglu_inference(self.mlp, hidden_states)
|
||||
hidden_states = fast_swiglu_inference(self.feed_forward, hidden_states)
|
||||
hidden_states += residual
|
||||
else:
|
||||
residual = hidden_states
|
||||
|
|
|
|||
|
|
@ -46,9 +46,9 @@ except:
|
|||
transformers_version = Version(transformers_version)
|
||||
if not transformers_version >= Version("4.45.0"):
|
||||
raise ImportError(
|
||||
f"Unsloth: Your transformers version of {transformers_version} does not support Gemma2.\n"
|
||||
f"The minimum required version is 4.42.3.\n"
|
||||
f'Try `pip install --upgrade "transformers>=4.42.3"`\n'
|
||||
f"Unsloth: Your transformers version of {transformers_version} does not support Granite.\n"
|
||||
f"The minimum required version is 4.45.0.\n"
|
||||
f'Try `pip install --upgrade "transformers>=4.45.0"`\n'
|
||||
f"to obtain the latest transformers build, then restart this session."
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2326,7 +2326,7 @@ class FastLlamaModel:
|
|||
attn_implementation = "eager",
|
||||
**kwargs,
|
||||
)
|
||||
model.fast_generate = model.generate
|
||||
model.fast_generate = make_fast_generate_wrapper(model.generate)
|
||||
model.fast_generate_batches = None
|
||||
else:
|
||||
from unsloth_zoo.vllm_utils import (
|
||||
|
|
@ -2600,6 +2600,7 @@ class FastLlamaModel:
|
|||
loftq_config = {},
|
||||
temporary_location = "_unsloth_temporary_saved_buffers",
|
||||
qat_scheme = None,
|
||||
ensure_weight_tying = False,
|
||||
**kwargs,
|
||||
):
|
||||
if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":
|
||||
|
|
@ -2629,6 +2630,7 @@ class FastLlamaModel:
|
|||
init_lora_weights = init_lora_weights,
|
||||
loftq_config = loftq_config,
|
||||
temporary_location = temporary_location,
|
||||
ensure_weight_tying = ensure_weight_tying,
|
||||
**kwargs,
|
||||
)
|
||||
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
|
||||
|
|
@ -2779,9 +2781,10 @@ class FastLlamaModel:
|
|||
type(init_lora_weights) is bool
|
||||
or init_lora_weights == "gaussian"
|
||||
or init_lora_weights == "loftq"
|
||||
or init_lora_weights == "corda"
|
||||
):
|
||||
raise ValueError(
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].'
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
|
||||
)
|
||||
|
||||
if init_lora_weights == "loftq":
|
||||
|
|
@ -2952,6 +2955,7 @@ class FastLlamaModel:
|
|||
loftq_config = loftq_config,
|
||||
use_rslora = use_rslora,
|
||||
modules_to_save = modules_to_save,
|
||||
ensure_weight_tying = ensure_weight_tying,
|
||||
**kwargs,
|
||||
)
|
||||
if not SUPPORTS_LOFTQ:
|
||||
|
|
@ -3001,6 +3005,55 @@ class FastLlamaModel:
|
|||
|
||||
model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
|
||||
|
||||
if ensure_weight_tying:
|
||||
try:
|
||||
input_embeddings = model.get_input_embeddings()
|
||||
output_embeddings = model.get_output_embeddings()
|
||||
|
||||
if input_embeddings is not None and output_embeddings is not None:
|
||||
|
||||
def _retie_parameter(target_module, source_module):
|
||||
if not hasattr(source_module, "weight"):
|
||||
return
|
||||
weight = source_module.weight
|
||||
# Remove existing registration to avoid "attribute already exists"
|
||||
if "weight" in getattr(target_module, "_parameters", {}):
|
||||
target_module._parameters.pop("weight")
|
||||
if hasattr(target_module, "weight"):
|
||||
try:
|
||||
delattr(target_module, "weight")
|
||||
except Exception as exc:
|
||||
logger.warning_once(
|
||||
f"Unsloth: Could not delete existing weight attr during retie on "
|
||||
f"{type(target_module).__name__}: {exc}"
|
||||
)
|
||||
target_module.register_parameter("weight", weight)
|
||||
|
||||
# Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward)
|
||||
if hasattr(input_embeddings, "modules_to_save") and hasattr(
|
||||
output_embeddings, "modules_to_save"
|
||||
):
|
||||
if hasattr(
|
||||
input_embeddings.modules_to_save, "default"
|
||||
) and hasattr(output_embeddings.modules_to_save, "default"):
|
||||
_retie_parameter(
|
||||
output_embeddings.modules_to_save.default,
|
||||
input_embeddings.modules_to_save.default,
|
||||
)
|
||||
|
||||
# Tie original_module references as well if present
|
||||
if hasattr(input_embeddings, "original_module") and hasattr(
|
||||
output_embeddings, "original_module"
|
||||
):
|
||||
_retie_parameter(
|
||||
output_embeddings.original_module,
|
||||
input_embeddings.original_module,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning_once(
|
||||
f"Unsloth: Failed to ensure weight tying between embeddings and lm_head: {e}"
|
||||
)
|
||||
|
||||
if train_embed_tokens:
|
||||
print("Unsloth: Training embed_tokens in mixed precision to save VRAM")
|
||||
assert hasattr(model.get_input_embeddings(), "modules_to_save")
|
||||
|
|
|
|||
|
|
@ -151,8 +151,41 @@ class FastLanguageModel(FastLlamaModel):
|
|||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
# Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
|
||||
quantization_config = kwargs.get("quantization_config", None)
|
||||
if quantization_config is not None:
|
||||
if isinstance(quantization_config, dict):
|
||||
q_load_in_4bit = quantization_config.get("load_in_4bit", False)
|
||||
q_load_in_8bit = quantization_config.get("load_in_8bit", False)
|
||||
else:
|
||||
q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False)
|
||||
q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False)
|
||||
if q_load_in_4bit:
|
||||
load_in_4bit = True
|
||||
load_in_8bit = False
|
||||
if q_load_in_8bit:
|
||||
load_in_8bit = True
|
||||
load_in_4bit = False
|
||||
|
||||
# Login to allow private models
|
||||
token = hf_login(token)
|
||||
# Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
|
||||
if dtype is None and quantization_config is not None:
|
||||
bnb_compute_dtype = None
|
||||
if isinstance(quantization_config, dict):
|
||||
if quantization_config.get("load_in_4bit", False):
|
||||
bnb_compute_dtype = quantization_config.get(
|
||||
"bnb_4bit_compute_dtype", None
|
||||
)
|
||||
else:
|
||||
if getattr(quantization_config, "load_in_4bit", False):
|
||||
bnb_compute_dtype = getattr(
|
||||
quantization_config, "bnb_4bit_compute_dtype", None
|
||||
)
|
||||
if isinstance(bnb_compute_dtype, str):
|
||||
bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None)
|
||||
if isinstance(bnb_compute_dtype, torch.dtype):
|
||||
dtype = bnb_compute_dtype
|
||||
if load_in_8bit or full_finetuning or qat_scheme is not None:
|
||||
return FastModel.from_pretrained(
|
||||
model_name = model_name,
|
||||
|
|
@ -204,6 +237,17 @@ class FastLanguageModel(FastLlamaModel):
|
|||
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
|
||||
"You can do this in a terminal via `pip install vllm`"
|
||||
)
|
||||
if DEVICE_TYPE_TORCH == "cuda":
|
||||
for i in range(DEVICE_COUNT):
|
||||
# [TODO] DGX Spark vLLM breaks
|
||||
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
|
||||
print(
|
||||
"Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
|
||||
"Defaulting to native Unsloth inference."
|
||||
)
|
||||
fast_inference = False
|
||||
break
|
||||
|
||||
# [TODO] For now fast_inference only works with fast_inference ie vLLM
|
||||
if load_in_fp8 != False:
|
||||
if not fast_inference:
|
||||
|
|
@ -531,11 +575,17 @@ class FastLanguageModel(FastLlamaModel):
|
|||
if fast_inference:
|
||||
fast_inference, model_name = fast_inference_setup(model_name, model_config)
|
||||
|
||||
load_in_4bit_kwargs = load_in_4bit
|
||||
load_in_8bit_kwargs = load_in_8bit
|
||||
if quantization_config is not None and not fast_inference:
|
||||
load_in_4bit_kwargs = False
|
||||
load_in_8bit_kwargs = False
|
||||
|
||||
model, tokenizer = dispatch_model.from_pretrained(
|
||||
model_name = model_name,
|
||||
max_seq_length = max_seq_length,
|
||||
dtype = _get_dtype(dtype),
|
||||
load_in_4bit = load_in_4bit,
|
||||
load_in_4bit = load_in_4bit_kwargs,
|
||||
token = token,
|
||||
device_map = device_map,
|
||||
rope_scaling = rope_scaling,
|
||||
|
|
@ -572,22 +622,30 @@ class FastLanguageModel(FastLlamaModel):
|
|||
)
|
||||
|
||||
if load_in_4bit:
|
||||
# Fix up bitsandbytes config
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
# Fix up bitsandbytes config, but respect user-provided quantization_config
|
||||
if quantization_config is None:
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
else:
|
||||
if hasattr(quantization_config, "to_dict"):
|
||||
model.config.update(
|
||||
{"quantization_config": quantization_config.to_dict()}
|
||||
)
|
||||
elif isinstance(quantization_config, dict):
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
|
||||
if load_in_fp8 != False:
|
||||
_tag_model_with_fp8_torchao_config(model, fp8_mode)
|
||||
|
|
@ -679,12 +737,45 @@ class FastModel(FastBaseModel):
|
|||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
# Respect user-provided quantization_config (e.g. BitsAndBytesConfig)
|
||||
quantization_config = kwargs.get("quantization_config", None)
|
||||
if quantization_config is not None:
|
||||
if isinstance(quantization_config, dict):
|
||||
q_load_in_4bit = quantization_config.get("load_in_4bit", False)
|
||||
q_load_in_8bit = quantization_config.get("load_in_8bit", False)
|
||||
else:
|
||||
q_load_in_4bit = getattr(quantization_config, "load_in_4bit", False)
|
||||
q_load_in_8bit = getattr(quantization_config, "load_in_8bit", False)
|
||||
if q_load_in_4bit:
|
||||
load_in_4bit = True
|
||||
load_in_8bit = False
|
||||
if q_load_in_8bit:
|
||||
load_in_8bit = True
|
||||
load_in_4bit = False
|
||||
|
||||
# Login to allow private models
|
||||
token = hf_login(token)
|
||||
if whisper_language is not None:
|
||||
assert type(whisper_language) is str
|
||||
if whisper_task is not None:
|
||||
assert type(whisper_task) is str
|
||||
# Align dtype with bnb_4bit_compute_dtype if provided and dtype is unset.
|
||||
if dtype is None and quantization_config is not None:
|
||||
bnb_compute_dtype = None
|
||||
if isinstance(quantization_config, dict):
|
||||
if quantization_config.get("load_in_4bit", False):
|
||||
bnb_compute_dtype = quantization_config.get(
|
||||
"bnb_4bit_compute_dtype", None
|
||||
)
|
||||
else:
|
||||
if getattr(quantization_config, "load_in_4bit", False):
|
||||
bnb_compute_dtype = getattr(
|
||||
quantization_config, "bnb_4bit_compute_dtype", None
|
||||
)
|
||||
if isinstance(bnb_compute_dtype, str):
|
||||
bnb_compute_dtype = getattr(torch, bnb_compute_dtype, None)
|
||||
if isinstance(bnb_compute_dtype, torch.dtype):
|
||||
dtype = bnb_compute_dtype
|
||||
SUPPORTS_BFLOAT16 = is_bfloat16_supported()
|
||||
if dtype is None:
|
||||
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
|
||||
|
|
@ -744,6 +835,17 @@ class FastModel(FastBaseModel):
|
|||
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
|
||||
"You can do this in a terminal via `pip install vllm`"
|
||||
)
|
||||
if DEVICE_TYPE_TORCH == "cuda":
|
||||
for i in range(DEVICE_COUNT):
|
||||
# [TODO] DGX Spark vLLM breaks
|
||||
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
|
||||
print(
|
||||
"Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
|
||||
"Defaulting to native Unsloth inference."
|
||||
)
|
||||
fast_inference = False
|
||||
break
|
||||
|
||||
# [TODO] For now fast_inference only works with fast_inference ie vLLM
|
||||
if load_in_fp8 != False:
|
||||
if not fast_inference:
|
||||
|
|
@ -1147,12 +1249,18 @@ class FastModel(FastBaseModel):
|
|||
if auto_model is None:
|
||||
auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM
|
||||
|
||||
load_in_4bit_kwargs = load_in_4bit
|
||||
load_in_8bit_kwargs = load_in_8bit
|
||||
if quantization_config is not None and not fast_inference:
|
||||
load_in_4bit_kwargs = False
|
||||
load_in_8bit_kwargs = False
|
||||
|
||||
model, tokenizer = FastBaseModel.from_pretrained(
|
||||
model_name = model_name,
|
||||
max_seq_length = max_seq_length,
|
||||
dtype = _get_dtype(dtype),
|
||||
load_in_4bit = load_in_4bit,
|
||||
load_in_8bit = load_in_8bit,
|
||||
load_in_4bit = load_in_4bit_kwargs,
|
||||
load_in_8bit = load_in_8bit_kwargs,
|
||||
load_in_16bit = load_in_16bit,
|
||||
full_finetuning = full_finetuning,
|
||||
token = token,
|
||||
|
|
@ -1198,22 +1306,30 @@ class FastModel(FastBaseModel):
|
|||
)
|
||||
|
||||
if load_in_4bit:
|
||||
# Fix up bitsandbytes config
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
# Fix up bitsandbytes config, but respect user-provided quantization_config
|
||||
if quantization_config is None:
|
||||
compute_dtype = dtype_from_config(model.config)
|
||||
quantization_config = {
|
||||
# Sometimes compute_dtype is not a string!!
|
||||
"bnb_4bit_compute_dtype": compute_dtype,
|
||||
"bnb_4bit_quant_type": "nf4",
|
||||
"bnb_4bit_use_double_quant": True,
|
||||
"llm_int8_enable_fp32_cpu_offload": False,
|
||||
"llm_int8_has_fp16_weight": False,
|
||||
"llm_int8_skip_modules": None,
|
||||
"llm_int8_threshold": 6.0,
|
||||
"load_in_4bit": True,
|
||||
"load_in_8bit": False,
|
||||
"quant_method": "bitsandbytes",
|
||||
}
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
else:
|
||||
if hasattr(quantization_config, "to_dict"):
|
||||
model.config.update(
|
||||
{"quantization_config": quantization_config.to_dict()}
|
||||
)
|
||||
elif isinstance(quantization_config, dict):
|
||||
model.config.update({"quantization_config": quantization_config})
|
||||
|
||||
if load_in_fp8 != False:
|
||||
_tag_model_with_fp8_torchao_config(model, fp8_mode)
|
||||
|
|
|
|||
|
|
@ -307,9 +307,9 @@ def MistralForCausalLM_fast_forward(
|
|||
RETURN_LOGITS = False
|
||||
|
||||
if not RETURN_LOGITS and labels is not None:
|
||||
n_items = kwargs.get("num_items_in_batch", None) or kwargs.get(
|
||||
"n_items", None
|
||||
)
|
||||
n_items = kwargs.get("num_items_in_batch", None)
|
||||
if n_items is None:
|
||||
n_items = kwargs.get("n_items", None)
|
||||
logit_softcapping = getattr(self.config, "final_logit_softcapping", 0)
|
||||
|
||||
# loss = fused_linear_cross_entropy(
|
||||
|
|
@ -363,11 +363,13 @@ def MistralForCausalLM_fast_forward(
|
|||
shift_labels,
|
||||
kwargs.get("packed_seq_lengths"),
|
||||
)
|
||||
n_items = kwargs.get("num_items_in_batch", None)
|
||||
if n_items is None:
|
||||
n_items = kwargs.get("n_items", None)
|
||||
loss = fast_cross_entropy_loss(
|
||||
logits = shift_logits,
|
||||
labels = shift_labels,
|
||||
n_items = kwargs.get("num_items_in_batch", None)
|
||||
or kwargs.get("n_items", None),
|
||||
n_items = n_items,
|
||||
)
|
||||
|
||||
if not return_dict:
|
||||
|
|
|
|||
|
|
@ -207,7 +207,7 @@ class FastQwen3MoeModel(FastQwen3Model):
|
|||
# https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py\
|
||||
import transformers.models.qwen3_moe.modeling_qwen3_moe
|
||||
|
||||
transformers.models.Qwen3Moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = (
|
||||
transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = (
|
||||
LlamaRotaryEmbedding
|
||||
)
|
||||
return
|
||||
|
|
@ -236,7 +236,7 @@ class FastQwen3MoeModel(FastQwen3Model):
|
|||
device_map = device_map,
|
||||
rope_scaling = rope_scaling,
|
||||
fix_tokenizer = fix_tokenizer,
|
||||
model_patcher = FastQwen3Model,
|
||||
model_patcher = FastQwen3MoeModel,
|
||||
tokenizer_name = tokenizer_name,
|
||||
trust_remote_code = trust_remote_code,
|
||||
**kwargs,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
|
|||
import inspect
|
||||
import os
|
||||
import re
|
||||
import torch
|
||||
from unsloth_zoo.compiler import create_new_function
|
||||
from unsloth_zoo.log import logger
|
||||
from unsloth_zoo.logging_utils import PatchRLStatistics
|
||||
|
|
@ -200,15 +199,15 @@ def PatchRL(FastLanguageModel):
|
|||
unwrap = "unwrap_model_for_generation"
|
||||
for trainer in trainers:
|
||||
try:
|
||||
current_trainer = eval(f"trl.trainer.{trainer}")
|
||||
current_trainer = getattr(trl.trainer, trainer)
|
||||
except:
|
||||
continue
|
||||
if hasattr(current_trainer, unwrap):
|
||||
try:
|
||||
exec(f"trl.trainer.{trainer}.{unwrap} = unsloth_{unwrap}")
|
||||
setattr(current_trainer, unwrap, unsloth_unwrap_model_for_generation)
|
||||
except:
|
||||
continue
|
||||
exec(f"Trainer.prediction_step=unsloth_prediction_step")
|
||||
Trainer.prediction_step = unsloth_prediction_step
|
||||
|
||||
|
||||
selective_log_softmax = RL_REPLACEMENTS["selective_log_softmax"]
|
||||
|
|
@ -227,6 +226,7 @@ import numpy as np
|
|||
from contextlib import nullcontext
|
||||
from torch.nn import functional as F
|
||||
import inspect
|
||||
import psutil
|
||||
from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling as TransformersDataCollatorForLanguageModeling
|
||||
from transformers.training_args import ParallelMode
|
||||
|
||||
|
|
@ -234,6 +234,10 @@ from transformers.training_args import ParallelMode
|
|||
# Also patches W&B since multiple runs must use wandb.finish()
|
||||
import functools
|
||||
from types import MethodType
|
||||
try:
|
||||
from unsloth_zoo.gradient_checkpointing import reset_unsloth_gradient_checkpointing_buffers
|
||||
except:
|
||||
def reset_unsloth_gradient_checkpointing_buffers(): pass
|
||||
def prepare_for_training_mode(f):
|
||||
@functools.wraps(f)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
|
|
@ -250,6 +254,11 @@ def prepare_for_training_mode(f):
|
|||
self.model.for_inference()
|
||||
elif _was_training is True and hasattr(self.model, "for_training"):
|
||||
self.model.for_training()
|
||||
# Reset gradient checkpointing buffers to free memory while staying ready for next run
|
||||
try:
|
||||
reset_unsloth_gradient_checkpointing_buffers()
|
||||
except:
|
||||
pass
|
||||
# Patch W&B to enable logging on future runs, otherwise it'll overwrite the first run
|
||||
try:
|
||||
import wandb
|
||||
|
|
@ -591,8 +600,12 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
" if args_max_seq_length is None and model_max_seq_length is not None:\n"
|
||||
" max_seq_length = model.max_seq_length\n"
|
||||
" if hasattr(args, 'max_seq_length'): args.max_seq_length = max_seq_length\n"
|
||||
" elif args_max_seq_length is not None and model_max_seq_length is not None:\n"
|
||||
" if args_max_seq_length > model_max_seq_length:\n"
|
||||
" print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but '\n"
|
||||
" 'the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n"
|
||||
" args.max_seq_length = model_max_seq_length\n"
|
||||
)
|
||||
" elif args_max_seq_length is not None and model_max_seq_length is not None:\n" " if args_max_seq_length > model_max_seq_length:\n" " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but \n" " the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" " args.max_seq_length = model_max_seq_length\n"
|
||||
extra_args += length_check
|
||||
|
||||
# At this point max_seq_length might be set, but trl is moving to max_length
|
||||
|
|
@ -713,6 +726,19 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
)
|
||||
RLTrainer_post += training_check
|
||||
|
||||
# Sync chat_template from processing_class to vLLM's tokenizer
|
||||
# This fixes base models that have custom chat templates applied after loading
|
||||
if "model" in call_args:
|
||||
vllm_chat_template_sync = (
|
||||
"if hasattr(self, 'llm') and self.llm is not None and hasattr(self.llm, 'get_tokenizer'):\n"
|
||||
" _vllm_tok = self.llm.get_tokenizer()\n"
|
||||
" _pc = getattr(self, 'processing_class', None) or getattr(self, 'tokenizer', None)\n"
|
||||
" if _vllm_tok is not None and _pc is not None and getattr(_pc, 'chat_template', None) is not None and getattr(_vllm_tok, 'chat_template', None) is None:\n"
|
||||
" _vllm_tok.chat_template = _pc.chat_template\n"
|
||||
"pass\n"
|
||||
)
|
||||
RLTrainer_post += vllm_chat_template_sync
|
||||
|
||||
# Edit optional metrics
|
||||
other_metrics_processor = ""
|
||||
if trainer_file in RL_METRICS_CHANGES:
|
||||
|
|
@ -845,7 +871,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
num_proc_check = (
|
||||
"if dataset_num_proc is None:\n"
|
||||
" import psutil\n"
|
||||
" dataset_num_proc = min(max(psutil.cpu_count()+4, 2), 64)\n"
|
||||
" dataset_num_proc = min(max((psutil.cpu_count() or 1)+4, 2), 64)\n"
|
||||
" memory_gb_left = psutil.virtual_memory().available / (1024**3)\n"
|
||||
" if memory_gb_left <= 4: dataset_num_proc = 1 # Too risky, so set to 1\n"
|
||||
" elif memory_gb_left <= 6: dataset_num_proc = min(2, dataset_num_proc)\n"
|
||||
|
|
@ -932,9 +958,9 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
if "temperature" in call_args:
|
||||
check_temperature = (
|
||||
"if temperature <= 0:\n"
|
||||
" raise MathError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n"
|
||||
" raise ValueError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n"
|
||||
"elif temperature >= 10:\n"
|
||||
" raise MathError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n"
|
||||
" raise ValueError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n"
|
||||
"\n"
|
||||
)
|
||||
extra_args += check_temperature
|
||||
|
|
@ -1022,10 +1048,10 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
|
||||
# Temporary patch _is_vlm to False
|
||||
# as of 0.22 it only exists in sfttrainer
|
||||
oriignal_is_vlm_text = "self._is_vlm = True"
|
||||
original_is_vlm_text = "self._is_vlm = True"
|
||||
new_is_vlm_text = "self._is_vlm = False"
|
||||
RLTrainer_source = RLTrainer_source.replace(
|
||||
oriignal_is_vlm_text, new_is_vlm_text
|
||||
original_is_vlm_text, new_is_vlm_text
|
||||
)
|
||||
|
||||
# Remove multiple doc strings
|
||||
|
|
|
|||
|
|
@ -68,11 +68,9 @@ import functools
|
|||
import os
|
||||
import gc
|
||||
import math
|
||||
import functools
|
||||
from typing import Optional, Tuple, List, Union
|
||||
import re, inspect, sys
|
||||
import contextlib
|
||||
import types
|
||||
|
||||
try:
|
||||
from huggingface_hub.utils import get_token
|
||||
|
|
@ -149,7 +147,7 @@ def unsloth_base_fast_generate(
|
|||
elif "input_ids" in kwargs:
|
||||
input_ids = kwargs["input_ids"]
|
||||
elif "input" in kwargs:
|
||||
input_ids = kwargs["input_ids"]
|
||||
input_ids = kwargs["input"]
|
||||
elif "input_features" in kwargs:
|
||||
input_ids = kwargs["input_features"]
|
||||
elif "input_embeds" in kwargs:
|
||||
|
|
@ -158,7 +156,7 @@ def unsloth_base_fast_generate(
|
|||
input_ids = kwargs["inputs"]
|
||||
else:
|
||||
key = next(iter(kwargs.keys()))
|
||||
if type(kwargs["key"]) is not torch.Tensor:
|
||||
if type(kwargs[key]) is not torch.Tensor:
|
||||
raise TypeError("Unsloth: You need to pass in input_ids to .generate!")
|
||||
input_ids = kwargs[key]
|
||||
assert type(input_ids) is torch.Tensor
|
||||
|
|
@ -531,6 +529,7 @@ class FastBaseModel:
|
|||
del kwargs["attn_implementation"]
|
||||
|
||||
bnb_config = None
|
||||
user_quantization_config = kwargs.get("quantization_config", None)
|
||||
if full_finetuning and (load_in_4bit or load_in_8bit):
|
||||
print(
|
||||
"Unsloth: You selected full finetuning support, but 4bit / 8bit is enabled - disabling LoRA / QLoRA."
|
||||
|
|
@ -598,7 +597,8 @@ class FastBaseModel:
|
|||
):
|
||||
pass
|
||||
else:
|
||||
kwargs["quantization_config"] = bnb_config
|
||||
if user_quantization_config is None:
|
||||
kwargs["quantization_config"] = bnb_config
|
||||
else:
|
||||
if auto_config is None:
|
||||
auto_config = AutoConfig.from_pretrained(
|
||||
|
|
@ -643,7 +643,8 @@ class FastBaseModel:
|
|||
)
|
||||
except:
|
||||
pass
|
||||
kwargs["quantization_config"] = quantization_config
|
||||
if user_quantization_config is None:
|
||||
kwargs["quantization_config"] = quantization_config
|
||||
|
||||
# Check if using forced float32 - we load it in bfloat16, then cast to float16!
|
||||
torch_dtype = dtype
|
||||
|
|
@ -675,7 +676,7 @@ class FastBaseModel:
|
|||
**kwargs,
|
||||
)
|
||||
if hasattr(model, "generate"):
|
||||
model.fast_generate = model.generate
|
||||
model.fast_generate = make_fast_generate_wrapper(model.generate)
|
||||
model.fast_generate_batches = error_out_no_vllm
|
||||
if offload_embedding:
|
||||
if bool(
|
||||
|
|
@ -718,9 +719,13 @@ class FastBaseModel:
|
|||
if full_finetuning:
|
||||
max_lora_rank = max(get_lora_supported_ranks())
|
||||
raise NotImplementedError(
|
||||
f"Unsloth: `fast_inference = True` does not yet support `full_finetuning = True`.\n"
|
||||
f"Use LoRA rank `r = {max_lora_rank}` as the closest replacement for full finetuning with Unsloth for RL."
|
||||
"Unsloth: `fast_inference=True` cannot be used together with `full_finetuning=True`.\n"
|
||||
"Reason: fast_inference is optimized for inference-only workflows and "
|
||||
"does not currently support full fine-tuning.\n"
|
||||
"Workaround: disable fast_inference, or use parameter-efficient fine-tuning "
|
||||
f"(e.g. LoRA with rank r={max_lora_rank})."
|
||||
)
|
||||
|
||||
model_config.model_name = model_name
|
||||
|
||||
if fast_inference:
|
||||
|
|
@ -936,6 +941,7 @@ class FastBaseModel:
|
|||
task_type = TaskType.CAUSAL_LM,
|
||||
temporary_location = "_unsloth_temporary_saved_buffers",
|
||||
qat_scheme = None,
|
||||
ensure_weight_tying = False, # [TODO] Add `ensure_weight_tying` for `modules_to_save` for vision models
|
||||
**kwargs,
|
||||
):
|
||||
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
|
||||
|
|
|
|||
|
|
@ -131,6 +131,13 @@ ALLOWED_QUANTS = {
|
|||
}
|
||||
|
||||
|
||||
def has_curl():
|
||||
return shutil.which("curl") is not None
|
||||
|
||||
|
||||
CURL_FLAG = "-DLLAMA_CURL=ON" if has_curl() else "-DLLAMA_CURL=OFF"
|
||||
|
||||
|
||||
def print_quantization_methods():
|
||||
for key, value in ALLOWED_QUANTS.items():
|
||||
print(f'"{key}" ==> {value}')
|
||||
|
|
@ -547,7 +554,7 @@ def unsloth_save_model(
|
|||
elif mb_found:
|
||||
sharded_ram_usage = int(mb_found.group(1)) * 1024 * 1024
|
||||
elif type(max_shard_size) is int:
|
||||
sharded_ram_usage = sharded_ram_usage
|
||||
sharded_ram_usage = max_shard_size
|
||||
|
||||
# Switch to our fast saving modules if it's a slow PC!
|
||||
n_cpus = psutil.cpu_count(logical = False)
|
||||
|
|
@ -872,15 +879,16 @@ def install_llama_cpp_make_non_blocking():
|
|||
IS_CMAKE = False
|
||||
if check == 0:
|
||||
# Uses old MAKE
|
||||
n_jobs = max(int(psutil.cpu_count() * 1.5), 1)
|
||||
n_jobs = max(int((psutil.cpu_count() or 1) * 1.5), 1)
|
||||
full_command = ["make", "all", "-j" + str(n_jobs), "-C", "llama.cpp"]
|
||||
IS_CMAKE = False
|
||||
else:
|
||||
# Uses new CMAKE
|
||||
n_jobs = max(int(psutil.cpu_count()), 1) # Use less CPUs since 1.5x faster
|
||||
n_jobs = max(int(psutil.cpu_count() or 1), 1) # Use less CPUs since 1.5x faster
|
||||
check = os.system(
|
||||
"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON"
|
||||
f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}"
|
||||
)
|
||||
|
||||
if check != 0:
|
||||
raise RuntimeError(
|
||||
f"*** Unsloth: Failed compiling llama.cpp using os.system(...) with error {check}. Please report this ASAP!"
|
||||
|
|
@ -986,16 +994,17 @@ def install_llama_cpp_old(version = -10):
|
|||
# Try using MAKE
|
||||
commands = [
|
||||
"make clean -C llama.cpp",
|
||||
f"make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
f"make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp",
|
||||
]
|
||||
if try_execute(commands) == "CMAKE":
|
||||
# Instead use CMAKE
|
||||
commands = [
|
||||
"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON",
|
||||
f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",
|
||||
f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}",
|
||||
f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1)*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",
|
||||
"cp llama.cpp/build/bin/llama-* llama.cpp",
|
||||
"rm -rf llama.cpp/build",
|
||||
]
|
||||
|
||||
try_execute(commands)
|
||||
|
||||
# Check if successful
|
||||
|
|
@ -1031,14 +1040,14 @@ def install_llama_cpp_blocking(use_cuda = False):
|
|||
"make clean -C llama.cpp",
|
||||
# https://github.com/ggerganov/llama.cpp/issues/7062
|
||||
# Weirdly GPU conversion for GGUF breaks??
|
||||
# f"{use_cuda} make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
f"make all -j{psutil.cpu_count()*2} -C llama.cpp",
|
||||
# f"{use_cuda} make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp",
|
||||
f"make all -j{(psutil.cpu_count() or 1)*2} -C llama.cpp",
|
||||
]
|
||||
if try_execute(commands) == "CMAKE":
|
||||
# Instead use CMAKE
|
||||
commands = [
|
||||
"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF -DLLAMA_CURL=ON",
|
||||
f"cmake --build llama.cpp/build --config Release -j{psutil.cpu_count()*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",
|
||||
f"cmake llama.cpp -B llama.cpp/build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=OFF {CURL_FLAG}",
|
||||
f"cmake --build llama.cpp/build --config Release -j{(psutil.cpu_count() or 1)*2} --clean-first --target {' '.join(LLAMA_CPP_TARGETS)}",
|
||||
"cp llama.cpp/build/bin/llama-* llama.cpp",
|
||||
"rm -rf llama.cpp/build",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import collections
|
|||
import numpy as np
|
||||
import gc
|
||||
import subprocess
|
||||
import psutil
|
||||
|
||||
from unsloth_zoo.tokenizer_utils import (
|
||||
mean_of_trained_tokens,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
import logging
|
||||
import os
|
||||
import psutil
|
||||
import warnings
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
|
|
|||
|
|
@ -32,9 +32,6 @@ from ..utils.packing import (
|
|||
if HAS_FLASH_ATTENTION:
|
||||
from flash_attn import flash_attn_func, flash_attn_varlen_func
|
||||
HAS_XFORMERS = xformers is not None
|
||||
BlockDiagonalCausalMask = None
|
||||
if HAS_XFORMERS:
|
||||
BlockDiagonalCausalMask = xformers.attn_bias.BlockDiagonalCausalMask
|
||||
SDPA_HAS_GQA = "enable_gqa" in (scaled_dot_product_attention.__doc__ or "")
|
||||
|
||||
FLASH_VARLEN = "flash_varlen"
|
||||
|
|
@ -219,16 +216,10 @@ def run_attention(
|
|||
)
|
||||
|
||||
if config.n_groups != 1 and not requires_grad:
|
||||
if has_block:
|
||||
out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim)
|
||||
else:
|
||||
out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim)
|
||||
out = out.view(bsz, q_len, config.n_kv_heads, config.n_groups, head_dim)
|
||||
out = out.reshape(bsz, q_len, n_heads, head_dim)
|
||||
else:
|
||||
if has_block:
|
||||
out = out.view(bsz, q_len, n_heads, head_dim)
|
||||
else:
|
||||
out = out.view(bsz, q_len, n_heads, head_dim)
|
||||
out = out.view(bsz, q_len, n_heads, head_dim)
|
||||
return out
|
||||
else:
|
||||
local_mask = context.attention_mask
|
||||
|
|
|
|||
|
|
@ -19,7 +19,9 @@ def formatted_int(value: int) -> str:
|
|||
elif value < MILLION:
|
||||
return f"{float(value) / 1000:,.1f}K"
|
||||
elif value < BILLION:
|
||||
return f"{float(value) // 1000000:,.1f}M"
|
||||
return f"{float(value) / 1000000:,.1f}M"
|
||||
else:
|
||||
return f"{float(value) / 1000000000:,.1f}B"
|
||||
|
||||
|
||||
def get_model_info(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue