Merge branch 'main' into nightly
This commit is contained in:
Daniel Han
commit
3959006d19
22 changed files with 246 additions and 114 deletions
2
.github/FUNDING.yml
vendored
2
.github/FUNDING.yml
vendored
|
|
@ -3,7 +3,7 @@
|
|||
github: unslothai
|
||||
patreon: # Replace with a single Patreon username
|
||||
open_collective: # Replace with a single Open Collective username
|
||||
ko_fi: unsloth
|
||||
ko_fi: # unsloth
|
||||
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
||||
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
||||
liberapay: # Replace with a single Liberapay username
|
||||
|
|
|
|||
126
README.md
126
README.md
|
|
@ -1,6 +1,6 @@
|
|||
<div align="center">
|
||||
|
||||
<a href="https://docs.unsloth.ai"><picture>
|
||||
<a href="https://unsloth.ai/docs"><picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20white%20text.png">
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20black%20text.png">
|
||||
<img alt="unsloth logo" src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/unsloth%20logo%20black%20text.png" height="110" style="max-width: 100%;">
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
<a href="https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-Fine-tuning.ipynb"><img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/start free finetune button.png" width="154"></a>
|
||||
<a href="https://discord.com/invite/unsloth"><img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/Discord button.png" width="165"></a>
|
||||
<a href="https://docs.unsloth.ai"><img src="https://raw.githubusercontent.com/unslothai/unsloth/refs/heads/main/images/Documentation%20Button.png" width="137"></a>
|
||||
<a href="https://unsloth.ai/docs"><img src="https://raw.githubusercontent.com/unslothai/unsloth/refs/heads/main/images/Documentation%20Button.png" width="137"></a>
|
||||
|
||||
### Train gpt-oss, DeepSeek, Gemma, Qwen & Llama 2x faster with 70% less VRAM!
|
||||
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
## ✨ Train for Free
|
||||
|
||||
Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then export your trained model to GGUF, llama.cpp, Ollama, vLLM, SGLang or Hugging Face.
|
||||
Notebooks are beginner friendly. Read our [guide](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide). Add dataset, run, then deploy your trained model.
|
||||
|
||||
| Model | Free Notebooks | Performance | Memory use |
|
||||
|-----------|---------|--------|----------|
|
||||
|
|
@ -34,9 +34,9 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st
|
|||
| **Llama 3.2 Conversational** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.2_(1B_and_3B)-Conversational.ipynb) | 2x faster | 70% less |
|
||||
| **Orpheus-TTS (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Orpheus_(3B)-TTS.ipynb) | 1.5x faster | 50% less |
|
||||
|
||||
- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://docs.unsloth.ai/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), **[TTS](https://docs.unsloth.ai/get-started/unsloth-notebooks#text-to-speech-tts-notebooks)** & [Vision](https://docs.unsloth.ai/get-started/unsloth-notebooks#vision-multimodal-notebooks)
|
||||
- See [all our models](https://docs.unsloth.ai/get-started/all-our-models) and [all our notebooks](https://docs.unsloth.ai/get-started/unsloth-notebooks)
|
||||
- See detailed documentation for Unsloth [here](https://docs.unsloth.ai/)
|
||||
- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://unsloth.ai/docs/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), [TTS](https://unsloth.ai/docs/get-started/unsloth-notebooks#text-to-speech-tts-notebooks) & [Vision](https://unsloth.ai/docs/get-started/unsloth-notebooks#vision-multimodal-notebooks)
|
||||
- See [all our models](https://unsloth.ai/docs/get-started/unsloth-model-catalog) and [all our notebooks](https://unsloth.ai/docs/get-started/unsloth-notebooks)
|
||||
- See detailed documentation for Unsloth [here](https://unsloth.ai/docs)
|
||||
|
||||
## ⚡ Quickstart
|
||||
### Linux or WSL
|
||||
|
|
@ -44,33 +44,35 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st
|
|||
pip install unsloth
|
||||
```
|
||||
### Windows
|
||||
For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://docs.unsloth.ai/get-started/installing-+-updating/windows-installation).
|
||||
For Windows, `pip install unsloth` works only if you have Pytorch installed. Read our [Windows Guide](https://unsloth.ai/docs/get-started/install-and-update/windows-installation).
|
||||
|
||||
### Docker
|
||||
Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://docs.unsloth.ai/get-started/install-and-update/docker).
|
||||
Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth) ```unsloth/unsloth``` container. Read our [Docker Guide](https://unsloth.ai/docs/get-started/install-and-update/docker).
|
||||
|
||||
### Blackwell & DGX Spark
|
||||
For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://docs.unsloth.ai/basics/training-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://docs.unsloth.ai/new/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details.
|
||||
For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://unsloth.ai/docs/basics/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details.
|
||||
|
||||
## 🦥 Unsloth News
|
||||
- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://docs.unsloth.ai/new/3x-faster-training-packing)
|
||||
- **Ministral 3** by Mistral: Run Ministral 3 or fine-tune with vision/RL sodoku notebooks. [Guide](https://docs.unsloth.ai/new/ministral-3) • [Notebooks](https://docs.unsloth.ai/new/ministral-3#fine-tuningb)
|
||||
- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://docs.unsloth.ai/new/500k-context-length-fine-tuning)
|
||||
- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://docs.unsloth.ai/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://docs.unsloth.ai/new/deepseek-ocr-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb)
|
||||
- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://docs.unsloth.ai/new/how-to-train-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth)
|
||||
- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://docs.unsloth.ai/new/gpt-oss-reinforcement-learning)
|
||||
- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://docs.unsloth.ai/new/vision-reinforcement-learning-vlm-rl)
|
||||
- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://docs.unsloth.ai/new/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://docs.unsloth.ai/basics/gpt-oss). 20B works on 14GB VRAM. 120B on 65GB.
|
||||
- New RoPE & MLP **Triton Kernels** & **Padding Free + Packing**: 3x faster training & 30% less VRAM. [Blog](https://unsloth.ai/docs/new/3x-faster-training-packing)
|
||||
- **New Mistral**: Run Ministral 3 or Devstral 2 and fine-tune with vision/RL sodoku notebooks. [Guide](https://unsloth.ai/docs/models/ministral-3) • [Notebooks](https://unsloth.ai/docs/models/ministral-3#fine-tuning-ministral-3)
|
||||
- **500K Context**: Training a 20B model with >500K context is now possible on an 80GB GPU. [Blog](https://unsloth.ai/docs/new/500k-context-length-fine-tuning)
|
||||
- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://unsloth.ai/docs/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://unsloth.ai/docs/models/deepseek-ocr-how-to-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb)
|
||||
- **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://unsloth.ai/docs/new/how-to-fine-tune-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth)
|
||||
- **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/gpt-oss-reinforcement-learning)
|
||||
- **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/vision-reinforcement-learning-vlm-rl)
|
||||
- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://unsloth.ai/docs/models/gpt-oss-how-to-run-and-fine-tune). 20B works on 14GB VRAM. 120B on 65GB.
|
||||
|
||||
<details>
|
||||
<summary>Click for more news</summary>
|
||||
|
||||
- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://docs.unsloth.ai/new/quantization-aware-training-qat)
|
||||
- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://docs.unsloth.ai/new/memory-efficient-rl)
|
||||
- **Gemma 3n** by Google: [Read Blog](https://docs.unsloth.ai/basics/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339).
|
||||
- **[Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`.
|
||||
- **[Qwen3](https://docs.unsloth.ai/basics/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM.
|
||||
- Introducing **[Dynamic 2.0](https://docs.unsloth.ai/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot.
|
||||
- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://docs.unsloth.ai/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`.
|
||||
- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://unsloth.ai/docs/basics/quantization-aware-training-qat)
|
||||
- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/memory-efficient-rl)
|
||||
- **Gemma 3n** by Google: [Read Blog](https://unsloth.ai/docs/models/gemma-3-how-to-run-and-fine-tune/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339).
|
||||
- **[Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`.
|
||||
- **[Qwen3](https://unsloth.ai/docs/models/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM.
|
||||
- Introducing **[Dynamic 2.0](https://unsloth.ai/docs/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot.
|
||||
- [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://unsloth.ai/docs/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`.
|
||||
- 📣 [DeepSeek-R1](https://unsloth.ai/blog/deepseek-r1) - run or fine-tune them [with our guide](https://unsloth.ai/blog/deepseek-r1). All model uploads: [here](https://huggingface.co/collections/unsloth/deepseek-r1-all-versions-678e1c48f5d2fce87892ace5).
|
||||
- 📣 Introducing Long-context [Reasoning (GRPO)](https://unsloth.ai/blog/grpo) in Unsloth. Train your own reasoning model with just 5GB VRAM. Transform Llama, Phi, Mistral etc. into reasoning LLMs!
|
||||
- 📣 Introducing Unsloth [Dynamic 4-bit Quantization](https://unsloth.ai/blog/dynamic-4bit)! We dynamically opt not to quantize certain parameters and this greatly increases accuracy while only using <10% more VRAM than BnB 4-bit. See our collection on [Hugging Face here.](https://huggingface.co/collections/unsloth/unsloth-4-bit-dynamic-quants-67503bb873f89e15276c44e7)
|
||||
|
|
@ -84,27 +86,29 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](
|
|||
</details>
|
||||
|
||||
## 🔗 Links and Resources
|
||||
| Type | Links |
|
||||
| ------------------------------- | --------------------------------------- |
|
||||
| <img width="15" src="https://redditinc.com/hs-fs/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png" /> **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth)|
|
||||
| 📚 **Documentation & Wiki** | [Read Our Docs](https://docs.unsloth.ai) |
|
||||
| <img width="16" src="https://upload.wikimedia.org/wikipedia/commons/6/6f/Logo_of_Twitter.svg" /> **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai)|
|
||||
| 💾 **Installation** | [Pip & Docker Install](https://docs.unsloth.ai/get-started/installing-+-updating)|
|
||||
| 🔮 **Our Models** | [Unsloth Catalog](https://docs.unsloth.ai/get-started/all-our-models)|
|
||||
| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog)|
|
||||
| Type | Links |
|
||||
| ----------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ |
|
||||
| <img width="15" src="https://redditinc.com/hs-fs/hubfs/Reddit%20Inc/Brand/Reddit_Logo.png" /> **r/unsloth Reddit** | [Join Reddit community](https://reddit.com/r/unsloth) |
|
||||
| 📚 **Documentation & Wiki** | [Read Our Docs](https://unsloth.ai/docs) |
|
||||
| <img width="13" src="https://upload.wikimedia.org/wikipedia/commons/0/09/X_(formerly_Twitter)_logo_late_2025.svg" /> **Twitter (aka X)** | [Follow us on X](https://twitter.com/unslothai) |
|
||||
| 💾 **Installation** | [Pip & Docker Install](https://unsloth.ai/docs/get-started/install-and-update) |
|
||||
| 🔮 **Our Models** | [Unsloth Catalog](https://unsloth.ai/docs/get-started/unsloth-model-catalog) |
|
||||
| ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog) |
|
||||
|
||||
## ⭐ Key Features
|
||||
- Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training
|
||||
- Supports **all models** including [TTS](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://docs.unsloth.ai/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth.
|
||||
- The most efficient library for [Reinforcement Learning (RL)](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc.
|
||||
- **0% loss in accuracy** - no approximation methods - all exact.
|
||||
- Supports NVIDIA (since 2018), [AMD](https://docs.unsloth.ai/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc)
|
||||
- Works on **Linux**, WSL and **Windows**
|
||||
- All kernels written in [OpenAI's Triton](https://openai.com/index/triton/) language. Manual backprop engine.
|
||||
- If you trained a model with 🦥Unsloth, you can use this cool sticker! <img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/made with unsloth.png" width="200" align="center" />
|
||||
|
||||
* Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training
|
||||
* Supports **all models** including [TTS](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://unsloth.ai/docs/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth.
|
||||
* The most efficient library for [Reinforcement Learning (RL)](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc.
|
||||
* **0% loss in accuracy** - no approximation methods - all exact.
|
||||
* Export and [deploy your model](https://unsloth.ai/docs/basics/inference-and-deployment) to GGUF, llama.cpp, vLLM, SGLang and Hugging Face.
|
||||
* Supports NVIDIA (since 2018), [AMD](https://unsloth.ai/docs/get-started/install-and-update/amd) and Intel GPUs. Minimum CUDA Capability 7.0 (V100, T4, Titan V, RTX 20, 30, 40x, A100, H100, L40 etc)
|
||||
* Works on **Linux**, WSL and **Windows**
|
||||
* All kernels written in OpenAI's Triton language. Manual backprop engine.
|
||||
* If you trained a model with 🦥Unsloth, you can use this cool sticker! <img src="https://raw.githubusercontent.com/unslothai/unsloth/main/images/made with unsloth.png" width="200" align="center" />
|
||||
|
||||
## 💾 Install Unsloth
|
||||
You can also see our docs for more detailed installation and updating instructions [here](https://docs.unsloth.ai/get-started/installing-+-updating).
|
||||
You can also see our docs for more detailed installation and updating instructions [here](https://unsloth.ai/docs/get-started/install-and-update).
|
||||
|
||||
Unsloth supports Python 3.13 or lower.
|
||||
|
||||
|
|
@ -124,7 +128,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions.
|
|||
You should install the latest driver for your GPU. Download drivers here: [NVIDIA GPU Driver](https://www.nvidia.com/Download/index.aspx).
|
||||
|
||||
3. **Install Visual Studio C++:**
|
||||
You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://docs.unsloth.ai/get-started/installing-+-updating).
|
||||
You will need Visual Studio, with C++ installed. By default, C++ is not installed with [Visual Studio](https://visualstudio.microsoft.com/vs/community/), so make sure you select all of the C++ options. Also select options for Windows 10/11 SDK. For detailed instructions with options, see [here](https://unsloth.ai/docs/get-started/install-and-update/windows-installation#method-3-windows-directly).
|
||||
|
||||
5. **Install CUDA Toolkit:**
|
||||
Follow the instructions to install [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit-archive).
|
||||
|
|
@ -139,19 +143,7 @@ See [here](#advanced-pip-installation) for advanced pip install instructions.
|
|||
pip install unsloth
|
||||
```
|
||||
|
||||
#### Notes
|
||||
To run Unsloth directly on Windows:
|
||||
- Install Triton from this Windows fork and follow the instructions [here](https://github.com/woct0rdho/triton-windows) (be aware that the Windows fork requires PyTorch >= 2.4 and CUDA 12)
|
||||
- In the `SFTConfig`, set `dataset_num_proc=1` to avoid a crashing issue:
|
||||
```python
|
||||
SFTConfig(
|
||||
dataset_num_proc=1,
|
||||
...
|
||||
)
|
||||
```
|
||||
|
||||
#### Advanced/Troubleshooting
|
||||
|
||||
For **advanced installation instructions** or if you see weird errors during installations:
|
||||
|
||||
First try using an isolated environment via then `pip install unsloth`
|
||||
|
|
@ -268,7 +260,7 @@ print(f'pip install --upgrade pip && pip install --no-deps git+https://github.co
|
|||
```
|
||||
### Docker Installation
|
||||
You can use our pre-built Docker container with all dependencies to use Unsloth instantly with no setup required.
|
||||
[Read our guide](https://docs.unsloth.ai/get-started/install-and-update/docker).
|
||||
[Read our guide](https://unsloth.ai/docs/get-started/install-and-update/docker).
|
||||
|
||||
This container requires installing [NVIDIA's Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
|
||||
|
||||
|
|
@ -283,9 +275,9 @@ docker run -d -e JUPYTER_PASSWORD="mypassword" \
|
|||
Access Jupyter Lab at `http://localhost:8888` and start fine-tuning!
|
||||
|
||||
## 📜 Documentation
|
||||
- Go to our official [Documentation](https://docs.unsloth.ai) for [running models](https://docs.unsloth.ai/basics/running-and-saving-models), [saving to GGUF](https://docs.unsloth.ai/basics/running-and-saving-models/saving-to-gguf), [checkpointing](https://docs.unsloth.ai/basics/finetuning-from-last-checkpoint), [evaluation](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide#evaluation) and more!
|
||||
- Read our Guides for: [Fine-tuning](https://docs.unsloth.ai/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), [Vision](https://docs.unsloth.ai/basics/vision-fine-tuning) and [any model](https://docs.unsloth.ai/models/tutorials-how-to-fine-tune-and-run-llms).
|
||||
- We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code.
|
||||
* Go to our official [Documentation](https://unsloth.ai/docs) for [running models](https://unsloth.ai/docs/basics/inference-and-deployment), [saving to GGUF](https://unsloth.ai/docs/basics/inference-and-deployment/saving-to-gguf), [checkpointing](https://unsloth.ai/docs/basics/finetuning-from-last-checkpoint), [evaluation](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide#evaluation) and more!
|
||||
* Read our Guides for: [Fine-tuning](https://unsloth.ai/docs/get-started/fine-tuning-llms-guide), [Reinforcement Learning](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide), [Text-to-Speech (TTS)](https://unsloth.ai/docs/basics/text-to-speech-tts-fine-tuning), [Vision](https://unsloth.ai/docs/basics/vision-fine-tuning) and [any model](https://unsloth.ai/docs/models/tutorials-how-to-fine-tune-and-run-llms).
|
||||
* We support Huggingface's transformers, TRL, Trainer, Seq2SeqTrainer and Pytorch code.
|
||||
|
||||
Unsloth example code to fine-tune gpt-oss-20b:
|
||||
|
||||
|
|
@ -310,8 +302,9 @@ model, tokenizer = FastModel.from_pretrained(
|
|||
max_seq_length = 2048, # Choose any for long context!
|
||||
load_in_4bit = True, # 4-bit quantization. False = 16-bit LoRA.
|
||||
load_in_8bit = False, # 8-bit quantization
|
||||
load_in_16bit = False, # [NEW!] 16-bit LoRA
|
||||
load_in_16bit = False, # 16-bit LoRA
|
||||
full_finetuning = False, # Use for full fine-tuning.
|
||||
trust_remote_code = False, # Enable to support new models
|
||||
# token = "hf_...", # use one if using gated models
|
||||
)
|
||||
|
||||
|
|
@ -350,7 +343,7 @@ trainer = SFTTrainer(
|
|||
)
|
||||
trainer.train()
|
||||
|
||||
# Go to https://docs.unsloth.ai for advanced tips like
|
||||
# Go to https://unsloth.ai/docs for advanced tips like
|
||||
# (1) Saving to GGUF / merging to 16bit for vLLM or SGLang
|
||||
# (2) Continued training from a saved LoRA adapter
|
||||
# (3) Adding an evaluation loop / OOMs
|
||||
|
|
@ -359,14 +352,15 @@ trainer.train()
|
|||
|
||||
<a name="RL"></a>
|
||||
## 💡 Reinforcement Learning
|
||||
[RL](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) including [GRPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), **FP8** traning, DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth.
|
||||
Read our [Reinforcement Learning Guide](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters.
|
||||
[RL](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) including [GRPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), [**FP8** training](https://unsloth.ai/docs/new/fp8-reinforcement-learning), DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth.
|
||||
|
||||
Read our [Reinforcement Learning Guide](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://unsloth.ai/docs/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters.
|
||||
|
||||
List of RL notebooks:
|
||||
- gpt-oss GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-GRPO.ipynb)
|
||||
- Qwen2.5-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen2_5_7B_VL_GRPO.ipynb)
|
||||
- - ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- Qwen2.3-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_VL_(8B)-Vision-GRPO.ipynb)
|
||||
- Advanced Qwen3 GRPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb)
|
||||
- ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb)
|
||||
- ORPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-ORPO.ipynb)
|
||||
- DPO Zephyr notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Zephyr_(7B)-DPO.ipynb)
|
||||
- KTO notebook: [Link](https://colab.research.google.com/drive/1MRgGtLWuZX4ypSfGguFgC-IblTvO2ivM?usp=sharing)
|
||||
|
|
@ -426,6 +420,4 @@ You can cite the Unsloth repo as follows:
|
|||
- The [llama.cpp library](https://github.com/ggml-org/llama.cpp) that lets users save models with Unsloth
|
||||
- The Hugging Face team and their libraries: [transformers](https://github.com/huggingface/transformers) and [TRL](https://github.com/huggingface/trl)
|
||||
- The Pytorch and [Torch AO](https://github.com/unslothai/unsloth/pull/3391) team for their contributions
|
||||
- [Erik](https://github.com/erikwijmans) for his help adding [Apple's ML Cross Entropy](https://github.com/apple/ml-cross-entropy) in Unsloth
|
||||
- [Etherl](https://github.com/Etherll) for adding support for [TTS, diffusion and BERT models](https://github.com/unslothai/notebooks/pull/34)
|
||||
- And of course for every single person who has contributed or has used Unsloth!
|
||||
|
|
|
|||
|
|
@ -30,16 +30,19 @@ from .import_fixes import (
|
|||
check_fbgemm_gpu_version,
|
||||
torchvision_compatibility_check,
|
||||
fix_diffusers_warnings,
|
||||
fix_huggingface_hub,
|
||||
)
|
||||
|
||||
fix_message_factory_issue()
|
||||
check_fbgemm_gpu_version()
|
||||
torchvision_compatibility_check()
|
||||
fix_diffusers_warnings()
|
||||
fix_huggingface_hub()
|
||||
del fix_message_factory_issue
|
||||
del check_fbgemm_gpu_version
|
||||
del torchvision_compatibility_check
|
||||
del fix_diffusers_warnings
|
||||
del fix_huggingface_hub
|
||||
|
||||
# This check is critical because Unsloth optimizes these libraries by modifying
|
||||
# their code at import time. If they're imported first, the original (slower,
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ __all__ = [
|
|||
|
||||
import torch
|
||||
import functools
|
||||
import inspect
|
||||
from unsloth_zoo.utils import Version
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from packaging.version import Version as TrueVersion
|
|||
import re
|
||||
import logging
|
||||
import textwrap
|
||||
import warnings
|
||||
|
||||
# We cannot do from unsloth_zoo.log import logger since FBGEMM might cause seg faults.
|
||||
UNSLOTH_ENABLE_LOGGING = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") in (
|
||||
|
|
@ -97,6 +98,10 @@ if os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") != "1":
|
|||
sys.stderr = HidePrintMessage(sys.stderr)
|
||||
# https://github.com/pytorch/FBGEMM/blob/d99cd96490ec4aabac2ee95b1e76ea4dcfcfa628/fbgemm_gpu/experimental/gemm/triton_gemm/utils.py#L43-L52
|
||||
sys.stderr.add_filter("TMA benchmarks will be running")
|
||||
# Skipping import of cpp extensions due to incompatible torch version 2.9.0+cu128 for torchao version 0.15.0
|
||||
logging.getLogger("torchao").setLevel(logging.ERROR)
|
||||
# SyntaxWarning: invalid escape sequence '\.'
|
||||
warnings.filterwarnings("ignore", message = "invalid escape sequence", category = SyntaxWarning)
|
||||
|
||||
|
||||
# Fix up AttributeError: 'MessageFactory' object has no attribute 'GetPrototype'
|
||||
|
|
@ -539,3 +544,10 @@ def fix_executorch():
|
|||
def fix_diffusers_warnings():
|
||||
# Silence Flax classes are deprecated and will be removed in Diffusers v1.0.0.
|
||||
os.environ["DIFFUSERS_VERBOSITY"] = "error"
|
||||
|
||||
|
||||
def fix_huggingface_hub():
|
||||
# huggingface_hub.is_offline_mode got removed, so add it back
|
||||
import huggingface_hub
|
||||
if not hasattr(huggingface_hub, "is_offline_mode"):
|
||||
huggingface_hub.is_offline_mode = lambda: huggingface_hub.constants.HF_HUB_OFFLINE
|
||||
|
|
|
|||
|
|
@ -379,9 +379,22 @@ class LoRA_QKV(torch.autograd.Function):
|
|||
):
|
||||
dtype = X.dtype
|
||||
|
||||
Q = matmul_lora(X, QW, QW_quant, QA, QB, QS)
|
||||
K = matmul_lora(X, KW, KW_quant, KA, KB, KS)
|
||||
V = matmul_lora(X, VW, VW_quant, VA, VB, VS)
|
||||
# bitsandbytes 8-bit matmul expects 2D inputs.
|
||||
# TorchInductor/AOTAutograd fails on 3D tensors during backward,
|
||||
# so we explicitly flatten the sequence dimension.
|
||||
orig_shape = X.shape
|
||||
X_for_matmul = X
|
||||
if X.dim() == 3:
|
||||
X_for_matmul = X.view(-1, X.shape[-1])
|
||||
Q = matmul_lora(X_for_matmul, QW, QW_quant, QA, QB, QS)
|
||||
K = matmul_lora(X_for_matmul, KW, KW_quant, KA, KB, KS)
|
||||
V = matmul_lora(X_for_matmul, VW, VW_quant, VA, VB, VS)
|
||||
|
||||
# Restore original shape after matmul
|
||||
if len(orig_shape) == 3:
|
||||
Q = Q.view(orig_shape[0], orig_shape[1], -1)
|
||||
K = K.view(orig_shape[0], orig_shape[1], -1)
|
||||
V = V.view(orig_shape[0], orig_shape[1], -1)
|
||||
|
||||
ctx.custom_saved_tensors = (
|
||||
QW,
|
||||
|
|
|
|||
|
|
@ -312,8 +312,8 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
|
|||
_, n_heads_K, _, _ = K.shape
|
||||
|
||||
# Inplace rotary embedding is generally fine
|
||||
Q_out = Q.clone() if not Q.is_contiguous else Q
|
||||
K_out = K.clone() if not K.is_contiguous else K
|
||||
Q_out = Q.clone() if not Q.is_contiguous() else Q
|
||||
K_out = K.clone() if not K.is_contiguous() else K
|
||||
|
||||
if has_indices:
|
||||
# TRL's rotary indices are always in int32, so casting is just for safety
|
||||
|
|
@ -383,21 +383,21 @@ class Fast_RoPE_Embedding_QK(torch.autograd.Function):
|
|||
else ctx.cos.new_empty(1, dtype = torch.int32)
|
||||
)
|
||||
|
||||
# Inplace rotary embedding is generally fine
|
||||
dQ_out = dQ.clone() if not dQ.is_contiguous() else dQ
|
||||
dK_out = dK.clone() if not dK.is_contiguous() else dK
|
||||
|
||||
Q_batch_stride, Q_head_stride, Q_seq_stride = (
|
||||
dQ.stride(0),
|
||||
dQ.stride(1),
|
||||
dQ.stride(2),
|
||||
dQ_out.stride(0),
|
||||
dQ_out.stride(1),
|
||||
dQ_out.stride(2),
|
||||
)
|
||||
K_batch_stride, K_head_stride, K_seq_stride = (
|
||||
dK.stride(0),
|
||||
dK.stride(1),
|
||||
dK.stride(2),
|
||||
dK_out.stride(0),
|
||||
dK_out.stride(1),
|
||||
dK_out.stride(2),
|
||||
)
|
||||
|
||||
# Inplace rotary embedding is generally fine
|
||||
dQ_out = dQ.clone() if not dQ.is_contiguous else dQ
|
||||
dK_out = dK.clone() if not dK.is_contiguous else dK
|
||||
|
||||
with torch_gpu_device(dQ.device):
|
||||
_rope_embedding_QK[(batch * ctx.seq_len, ctx.n_heads_Q)](
|
||||
dQ_out,
|
||||
|
|
|
|||
|
|
@ -1981,9 +1981,10 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m
|
|||
type(init_lora_weights) is bool
|
||||
or init_lora_weights == "gaussian"
|
||||
or init_lora_weights == "loftq"
|
||||
or init_lora_weights == "corda"
|
||||
):
|
||||
raise ValueError(
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].'
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
|
||||
)
|
||||
|
||||
if init_lora_weights == "loftq":
|
||||
|
|
@ -2197,6 +2198,18 @@ def _prepare_model_for_qat(
|
|||
from torchao.quantization.granularity import PerGroup, PerAxis
|
||||
from torchao.quantization.qat import QATConfig
|
||||
|
||||
# Gemma3 models have issues with int8 embedding quantization due to their
|
||||
# large vocabulary size (262144). Auto-switch to int4 weight-only instead.
|
||||
if qat_scheme == "int8-int4":
|
||||
model_types = get_transformers_model_type(model.config)
|
||||
is_gemma3 = any("gemma3" in mt or "gemma_3" in mt for mt in model_types)
|
||||
if is_gemma3:
|
||||
print(
|
||||
"Unsloth: Gemma3 has a large vocabulary causing int8 embedding issues. "
|
||||
"Switching to int4 weight-only QAT for training stability."
|
||||
)
|
||||
qat_scheme = "int4"
|
||||
|
||||
if not isinstance(qat_scheme, TorchAOConfig):
|
||||
torchao_config: Optional[TorchAOConfig] = None
|
||||
if qat_scheme == "fp8-int4":
|
||||
|
|
|
|||
|
|
@ -344,8 +344,8 @@ def CohereAttention_fast_forward_inference(
|
|||
Kn = Kn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
|
||||
Vn = Vn.view(bsz, 1, n_kv_heads, head_dim).transpose(1, 2)
|
||||
if self.use_qk_norm:
|
||||
Q = fast_layernorm_inference(self.q_norm, Q, self.q_norm_out_weight)
|
||||
K = fast_layernorm_inference(self.k_norm, K, self.k_norm_out_weight)
|
||||
Qn = fast_layernorm_inference(self.q_norm, Qn, self.q_norm_out_weight)
|
||||
Kn = fast_layernorm_inference(self.k_norm, Kn, self.k_norm_out_weight)
|
||||
|
||||
# cos, sin = self.rotary_emb(Vn, seq_len = kv_seq_len)
|
||||
# Qn, Kn = inplace_rope_embedding(Qn, Kn, cos, sin, position_ids)
|
||||
|
|
@ -479,7 +479,7 @@ def CohereModel_fast_forward_inference(
|
|||
)
|
||||
)
|
||||
|
||||
hidden_states_mlp = fast_swiglu_inference(self.mlp, hidden_states)
|
||||
hidden_states_mlp = fast_swiglu_inference(decoder_layer.mlp, hidden_states)
|
||||
residual += hidden_states_attention
|
||||
residual += hidden_states_mlp
|
||||
hidden_states = residual
|
||||
|
|
|
|||
|
|
@ -456,9 +456,9 @@ def FalconH1DecoderLayer_fast_forward(
|
|||
# Fully Connected
|
||||
residual = hidden_states
|
||||
hidden_states = fast_rms_layernorm_inference(
|
||||
self.post_attention_layernorm, hidden_states
|
||||
self.pre_ff_layernorm, hidden_states
|
||||
)
|
||||
hidden_states = fast_swiglu_inference(self.mlp, hidden_states)
|
||||
hidden_states = fast_swiglu_inference(self.feed_forward, hidden_states)
|
||||
hidden_states += residual
|
||||
else:
|
||||
residual = hidden_states
|
||||
|
|
|
|||
|
|
@ -46,9 +46,9 @@ except:
|
|||
transformers_version = Version(transformers_version)
|
||||
if not transformers_version >= Version("4.45.0"):
|
||||
raise ImportError(
|
||||
f"Unsloth: Your transformers version of {transformers_version} does not support Gemma2.\n"
|
||||
f"The minimum required version is 4.42.3.\n"
|
||||
f'Try `pip install --upgrade "transformers>=4.42.3"`\n'
|
||||
f"Unsloth: Your transformers version of {transformers_version} does not support Granite.\n"
|
||||
f"The minimum required version is 4.45.0.\n"
|
||||
f'Try `pip install --upgrade "transformers>=4.45.0"`\n'
|
||||
f"to obtain the latest transformers build, then restart this session."
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -2600,6 +2600,7 @@ class FastLlamaModel:
|
|||
loftq_config = {},
|
||||
temporary_location = "_unsloth_temporary_saved_buffers",
|
||||
qat_scheme = None,
|
||||
ensure_weight_tying = False,
|
||||
**kwargs,
|
||||
):
|
||||
if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":
|
||||
|
|
@ -2629,6 +2630,7 @@ class FastLlamaModel:
|
|||
init_lora_weights = init_lora_weights,
|
||||
loftq_config = loftq_config,
|
||||
temporary_location = temporary_location,
|
||||
ensure_weight_tying = ensure_weight_tying,
|
||||
**kwargs,
|
||||
)
|
||||
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
|
||||
|
|
@ -2779,9 +2781,10 @@ class FastLlamaModel:
|
|||
type(init_lora_weights) is bool
|
||||
or init_lora_weights == "gaussian"
|
||||
or init_lora_weights == "loftq"
|
||||
or init_lora_weights == "corda"
|
||||
):
|
||||
raise ValueError(
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq"].'
|
||||
'Unsloth: `init_lora_weights` must be either [True, False, "gaussian", "loftq", "corda"].'
|
||||
)
|
||||
|
||||
if init_lora_weights == "loftq":
|
||||
|
|
@ -2952,6 +2955,7 @@ class FastLlamaModel:
|
|||
loftq_config = loftq_config,
|
||||
use_rslora = use_rslora,
|
||||
modules_to_save = modules_to_save,
|
||||
ensure_weight_tying = ensure_weight_tying,
|
||||
**kwargs,
|
||||
)
|
||||
if not SUPPORTS_LOFTQ:
|
||||
|
|
@ -3001,6 +3005,55 @@ class FastLlamaModel:
|
|||
|
||||
model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
|
||||
|
||||
if ensure_weight_tying:
|
||||
try:
|
||||
input_embeddings = model.get_input_embeddings()
|
||||
output_embeddings = model.get_output_embeddings()
|
||||
|
||||
if input_embeddings is not None and output_embeddings is not None:
|
||||
|
||||
def _retie_parameter(target_module, source_module):
|
||||
if not hasattr(source_module, "weight"):
|
||||
return
|
||||
weight = source_module.weight
|
||||
# Remove existing registration to avoid "attribute already exists"
|
||||
if "weight" in getattr(target_module, "_parameters", {}):
|
||||
target_module._parameters.pop("weight")
|
||||
if hasattr(target_module, "weight"):
|
||||
try:
|
||||
delattr(target_module, "weight")
|
||||
except Exception as exc:
|
||||
logger.warning_once(
|
||||
f"Unsloth: Could not delete existing weight attr during retie on "
|
||||
f"{type(target_module).__name__}: {exc}"
|
||||
)
|
||||
target_module.register_parameter("weight", weight)
|
||||
|
||||
# Tie trainable copies created by ModulesToSaveWrapper first (these are used in forward)
|
||||
if hasattr(input_embeddings, "modules_to_save") and hasattr(
|
||||
output_embeddings, "modules_to_save"
|
||||
):
|
||||
if hasattr(
|
||||
input_embeddings.modules_to_save, "default"
|
||||
) and hasattr(output_embeddings.modules_to_save, "default"):
|
||||
_retie_parameter(
|
||||
output_embeddings.modules_to_save.default,
|
||||
input_embeddings.modules_to_save.default,
|
||||
)
|
||||
|
||||
# Tie original_module references as well if present
|
||||
if hasattr(input_embeddings, "original_module") and hasattr(
|
||||
output_embeddings, "original_module"
|
||||
):
|
||||
_retie_parameter(
|
||||
output_embeddings.original_module,
|
||||
input_embeddings.original_module,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning_once(
|
||||
f"Unsloth: Failed to ensure weight tying between embeddings and lm_head: {e}"
|
||||
)
|
||||
|
||||
if train_embed_tokens:
|
||||
print("Unsloth: Training embed_tokens in mixed precision to save VRAM")
|
||||
assert hasattr(model.get_input_embeddings(), "modules_to_save")
|
||||
|
|
|
|||
|
|
@ -204,6 +204,17 @@ class FastLanguageModel(FastLlamaModel):
|
|||
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
|
||||
"You can do this in a terminal via `pip install vllm`"
|
||||
)
|
||||
if DEVICE_TYPE_TORCH == "cuda":
|
||||
for i in range(DEVICE_COUNT):
|
||||
# [TODO] DGX Spark vLLM breaks
|
||||
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
|
||||
print(
|
||||
"Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
|
||||
"Defaulting to native Unsloth inference."
|
||||
)
|
||||
fast_inference = False
|
||||
break
|
||||
|
||||
# [TODO] For now fast_inference only works with fast_inference ie vLLM
|
||||
if load_in_fp8 != False:
|
||||
if not fast_inference:
|
||||
|
|
@ -744,6 +755,17 @@ class FastModel(FastBaseModel):
|
|||
"Unsloth: Please install vLLM before enabling `fast_inference`!\n"
|
||||
"You can do this in a terminal via `pip install vllm`"
|
||||
)
|
||||
if DEVICE_TYPE_TORCH == "cuda":
|
||||
for i in range(DEVICE_COUNT):
|
||||
# [TODO] DGX Spark vLLM breaks
|
||||
if "NVIDIA GB10" in str(torch.cuda.get_device_name(i)).upper():
|
||||
print(
|
||||
"Unsloth: DGX Spark detected - `fast_inference=True` is currently broken as of January 2026.\n"
|
||||
"Defaulting to native Unsloth inference."
|
||||
)
|
||||
fast_inference = False
|
||||
break
|
||||
|
||||
# [TODO] For now fast_inference only works with fast_inference ie vLLM
|
||||
if load_in_fp8 != False:
|
||||
if not fast_inference:
|
||||
|
|
|
|||
|
|
@ -307,9 +307,9 @@ def MistralForCausalLM_fast_forward(
|
|||
RETURN_LOGITS = False
|
||||
|
||||
if not RETURN_LOGITS and labels is not None:
|
||||
n_items = kwargs.get("num_items_in_batch", None) or kwargs.get(
|
||||
"n_items", None
|
||||
)
|
||||
n_items = kwargs.get("num_items_in_batch", None)
|
||||
if n_items is None:
|
||||
n_items = kwargs.get("n_items", None)
|
||||
logit_softcapping = getattr(self.config, "final_logit_softcapping", 0)
|
||||
|
||||
# loss = fused_linear_cross_entropy(
|
||||
|
|
@ -363,11 +363,13 @@ def MistralForCausalLM_fast_forward(
|
|||
shift_labels,
|
||||
kwargs.get("packed_seq_lengths"),
|
||||
)
|
||||
n_items = kwargs.get("num_items_in_batch", None)
|
||||
if n_items is None:
|
||||
n_items = kwargs.get("n_items", None)
|
||||
loss = fast_cross_entropy_loss(
|
||||
logits = shift_logits,
|
||||
labels = shift_labels,
|
||||
n_items = kwargs.get("num_items_in_batch", None)
|
||||
or kwargs.get("n_items", None),
|
||||
n_items = n_items,
|
||||
)
|
||||
|
||||
if not return_dict:
|
||||
|
|
|
|||
|
|
@ -207,7 +207,7 @@ class FastQwen3MoeModel(FastQwen3Model):
|
|||
# https://github.com/huggingface/transformers/blob/v4.37.2/src/transformers/models/llama/modeling_llama.py\
|
||||
import transformers.models.qwen3_moe.modeling_qwen3_moe
|
||||
|
||||
transformers.models.Qwen3Moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = (
|
||||
transformers.models.qwen3_moe.modeling_qwen3_moe.Qwen3MoeRotaryEmbedding = (
|
||||
LlamaRotaryEmbedding
|
||||
)
|
||||
return
|
||||
|
|
@ -236,7 +236,7 @@ class FastQwen3MoeModel(FastQwen3Model):
|
|||
device_map = device_map,
|
||||
rope_scaling = rope_scaling,
|
||||
fix_tokenizer = fix_tokenizer,
|
||||
model_patcher = FastQwen3Model,
|
||||
model_patcher = FastQwen3MoeModel,
|
||||
tokenizer_name = tokenizer_name,
|
||||
trust_remote_code = trust_remote_code,
|
||||
**kwargs,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union
|
|||
import inspect
|
||||
import os
|
||||
import re
|
||||
import torch
|
||||
from unsloth_zoo.compiler import create_new_function
|
||||
from unsloth_zoo.log import logger
|
||||
from unsloth_zoo.logging_utils import PatchRLStatistics
|
||||
|
|
@ -227,6 +226,7 @@ import numpy as np
|
|||
from contextlib import nullcontext
|
||||
from torch.nn import functional as F
|
||||
import inspect
|
||||
import psutil
|
||||
from transformers import DataCollatorForSeq2Seq, DataCollatorForLanguageModeling as TransformersDataCollatorForLanguageModeling
|
||||
from transformers.training_args import ParallelMode
|
||||
|
||||
|
|
@ -559,8 +559,12 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
" if args_max_seq_length is None and model_max_seq_length is not None:\n"
|
||||
" max_seq_length = model.max_seq_length\n"
|
||||
" if hasattr(args, 'max_seq_length'): args.max_seq_length = max_seq_length\n"
|
||||
" elif args_max_seq_length is not None and model_max_seq_length is not None:\n"
|
||||
" if args_max_seq_length > model_max_seq_length:\n"
|
||||
" print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but '\n"
|
||||
" 'the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n"
|
||||
" args.max_seq_length = model_max_seq_length\n"
|
||||
)
|
||||
" elif args_max_seq_length is not None and model_max_seq_length is not None:\n" " if args_max_seq_length > model_max_seq_length:\n" " print('Unsloth: You set `max_seq_length` as ' + str(args_max_seq_length) + ' but \n" " the maximum the model supports is ' + str(model_max_seq_length) + '. We shall reduce it.')\n" " args.max_seq_length = model_max_seq_length\n"
|
||||
extra_args += length_check
|
||||
|
||||
# At this point max_seq_length might be set, but trl is moving to max_length
|
||||
|
|
@ -900,9 +904,9 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
if "temperature" in call_args:
|
||||
check_temperature = (
|
||||
"if temperature <= 0:\n"
|
||||
" raise MathError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n"
|
||||
" raise ValueError('Unsloth: Please set a positive non-zero temperature since your results will be wrong.')\n"
|
||||
"elif temperature >= 10:\n"
|
||||
" raise MathError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n"
|
||||
" raise ValueError('Unsloth: Please set a positive non-zero temperature less than 10, since sampling will be quite erratic.')\n"
|
||||
"\n"
|
||||
)
|
||||
extra_args += check_temperature
|
||||
|
|
|
|||
|
|
@ -861,13 +861,19 @@ def grpo_trainer_compute_loss(function_name, function):
|
|||
else torch.tensor(0.0, device = self.model.device)
|
||||
)
|
||||
self._metrics[mode]["sampling/importance_sampling_ratio/min"].append(
|
||||
nanmin(self.accelerator.gather(min_importance_sampling_ratio)).item()
|
||||
self.accelerator.gather(min_importance_sampling_ratio)
|
||||
.nan_to_num(nan = float("inf"))
|
||||
.min()
|
||||
.item()
|
||||
)
|
||||
self._metrics[mode]["sampling/importance_sampling_ratio/mean"].append(
|
||||
self.accelerator.gather(mean_importance_sampling_ratio).nanmean().item()
|
||||
)
|
||||
self._metrics[mode]["sampling/importance_sampling_ratio/max"].append(
|
||||
nanmax(self.accelerator.gather(max_importance_sampling_ratio)).item()
|
||||
self.accelerator.gather(max_importance_sampling_ratio)
|
||||
.nan_to_num(nan = float("-inf"))
|
||||
.max()
|
||||
.item()
|
||||
)
|
||||
|
||||
return loss
|
||||
|
|
@ -949,11 +955,15 @@ def openenv_vllm_reload_weights():
|
|||
return
|
||||
if Version(importlib_version("trl")) < Version("0.26.0"):
|
||||
return
|
||||
|
||||
try:
|
||||
import trl.experimental.openenv.utils as openenv_utils
|
||||
import trl.experimental.openenv as openenv
|
||||
except ImportError as e:
|
||||
logger.info(f"Unsloth: Failed to import trl openenv: {e}")
|
||||
logger.info(
|
||||
"Unsloth: trl.experimental.openenv not available — skipping RL openenv patches."
|
||||
)
|
||||
return
|
||||
|
||||
src = inspect.getsource(openenv_utils.generate_rollout_completions)
|
||||
|
|
|
|||
|
|
@ -68,11 +68,9 @@ import functools
|
|||
import os
|
||||
import gc
|
||||
import math
|
||||
import functools
|
||||
from typing import Optional, Tuple, List, Union
|
||||
import re, inspect, sys
|
||||
import contextlib
|
||||
import types
|
||||
|
||||
try:
|
||||
from huggingface_hub.utils import get_token
|
||||
|
|
@ -149,7 +147,7 @@ def unsloth_base_fast_generate(
|
|||
elif "input_ids" in kwargs:
|
||||
input_ids = kwargs["input_ids"]
|
||||
elif "input" in kwargs:
|
||||
input_ids = kwargs["input_ids"]
|
||||
input_ids = kwargs["input"]
|
||||
elif "input_features" in kwargs:
|
||||
input_ids = kwargs["input_features"]
|
||||
elif "input_embeds" in kwargs:
|
||||
|
|
@ -158,7 +156,7 @@ def unsloth_base_fast_generate(
|
|||
input_ids = kwargs["inputs"]
|
||||
else:
|
||||
key = next(iter(kwargs.keys()))
|
||||
if type(kwargs["key"]) is not torch.Tensor:
|
||||
if type(kwargs[key]) is not torch.Tensor:
|
||||
raise TypeError("Unsloth: You need to pass in input_ids to .generate!")
|
||||
input_ids = kwargs[key]
|
||||
assert type(input_ids) is torch.Tensor
|
||||
|
|
@ -718,9 +716,13 @@ class FastBaseModel:
|
|||
if full_finetuning:
|
||||
max_lora_rank = max(get_lora_supported_ranks())
|
||||
raise NotImplementedError(
|
||||
f"Unsloth: `fast_inference = True` does not yet support `full_finetuning = True`.\n"
|
||||
f"Use LoRA rank `r = {max_lora_rank}` as the closest replacement for full finetuning with Unsloth for RL."
|
||||
"Unsloth: `fast_inference=True` cannot be used together with `full_finetuning=True`.\n"
|
||||
"Reason: fast_inference is optimized for inference-only workflows and "
|
||||
"does not currently support full fine-tuning.\n"
|
||||
"Workaround: disable fast_inference, or use parameter-efficient fine-tuning "
|
||||
f"(e.g. LoRA with rank r={max_lora_rank})."
|
||||
)
|
||||
|
||||
model_config.model_name = model_name
|
||||
|
||||
if fast_inference:
|
||||
|
|
@ -936,6 +938,7 @@ class FastBaseModel:
|
|||
task_type = TaskType.CAUSAL_LM,
|
||||
temporary_location = "_unsloth_temporary_saved_buffers",
|
||||
qat_scheme = None,
|
||||
ensure_weight_tying = False, # [TODO] Add `ensure_weight_tying` for `modules_to_save` for vision models
|
||||
**kwargs,
|
||||
):
|
||||
if os.environ.get("UNSLOTH_ENABLE_FULL_FINETUNING", "0") == "1":
|
||||
|
|
|
|||
|
|
@ -547,7 +547,7 @@ def unsloth_save_model(
|
|||
elif mb_found:
|
||||
sharded_ram_usage = int(mb_found.group(1)) * 1024 * 1024
|
||||
elif type(max_shard_size) is int:
|
||||
sharded_ram_usage = sharded_ram_usage
|
||||
sharded_ram_usage = max_shard_size
|
||||
|
||||
# Switch to our fast saving modules if it's a slow PC!
|
||||
n_cpus = psutil.cpu_count(logical = False)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import collections
|
|||
import numpy as np
|
||||
import gc
|
||||
import subprocess
|
||||
import psutil
|
||||
|
||||
from unsloth_zoo.tokenizer_utils import (
|
||||
mean_of_trained_tokens,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
|
||||
import logging
|
||||
import os
|
||||
import psutil
|
||||
import warnings
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
|
|
|||
|
|
@ -19,7 +19,9 @@ def formatted_int(value: int) -> str:
|
|||
elif value < MILLION:
|
||||
return f"{float(value) / 1000:,.1f}K"
|
||||
elif value < BILLION:
|
||||
return f"{float(value) // 1000000:,.1f}M"
|
||||
return f"{float(value) / 1000000:,.1f}M"
|
||||
else:
|
||||
return f"{float(value) / 1000000000:,.1f}B"
|
||||
|
||||
|
||||
def get_model_info(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue