From 74a9fc808c559cddd30644df7bccedb746cb3cfc Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Thu, 27 Nov 2025 20:49:47 -0800 Subject: [PATCH 1/7] Update README.md --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fd28429f85..04ce316f5a 100644 --- a/README.md +++ b/README.md @@ -50,23 +50,23 @@ Use our official [Unsloth Docker image](https://hub.docker.com/r/unsloth/unsloth For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide](https://docs.unsloth.ai/basics/training-llms-with-blackwell-rtx-50-series-and-unsloth) and [DGX Spark Guide](https://docs.unsloth.ai/new/fine-tuning-llms-with-nvidia-dgx-spark-and-unsloth) for more details. ## 🦥 Unsloth News -- **DeepSeek-OCR**: Fine-tune to improve DeepSeek-OCR's language understanding by 89%. [Guide](https://docs.unsloth.ai/new/deepseek-ocr-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) +- **FP8 Reinforcement Learning**: You can now do FP8 GRPO on consumer GPUs. [Blog](https://docs.unsloth.ai/new/fp8-reinforcement-learning) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) +- **DeepSeek-OCR**: Fine-tune to improve language understanding by 89%. [Guide](https://docs.unsloth.ai/new/deepseek-ocr-run-and-fine-tune) • [Notebook](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) - **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://docs.unsloth.ai/new/how-to-train-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth) - **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://docs.unsloth.ai/new/gpt-oss-reinforcement-learning) - **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://docs.unsloth.ai/new/vision-reinforcement-learning-vlm-rl) - **Quantization-Aware Training**: We collabed with Pytorch for QAT, recovering as much 70% accuracy. [Read blog](https://docs.unsloth.ai/new/quantization-aware-training-qat) -- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://docs.unsloth.ai/new/memory-efficient-rl) - **gpt-oss** by OpenAI: For details on [Unsloth Flex Attention](https://docs.unsloth.ai/new/long-context-gpt-oss-training), long-context training, bug fixes, [Read our Guide](https://docs.unsloth.ai/basics/gpt-oss). 20B works on a 14GB GPU and 120B on 65GB VRAM. [gpt-oss uploads](https://huggingface.co/collections/unsloth/gpt-oss-6892433695ce0dee42f31681). + +
+ Click for more news + +- **Memory-efficient RL**: We're introducing even better RL. Our new kernels & algos allows faster RL with 50% less VRAM & 10× more context. [Read blog](https://docs.unsloth.ai/new/memory-efficient-rl) - **Gemma 3n** by Google: [Read Blog](https://docs.unsloth.ai/basics/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339). - **[Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`. - **[Qwen3](https://docs.unsloth.ai/basics/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM. - Introducing **[Dynamic 2.0](https://docs.unsloth.ai/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & Aider Polyglot. - [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (TTS, BERT, Mamba), FFT, etc. [MultiGPU](https://docs.unsloth.ai/basics/multi-gpu-training-with-unsloth) coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`. - -
- Click for more news - - - 📣 [DeepSeek-R1](https://unsloth.ai/blog/deepseek-r1) - run or fine-tune them [with our guide](https://unsloth.ai/blog/deepseek-r1). All model uploads: [here](https://huggingface.co/collections/unsloth/deepseek-r1-all-versions-678e1c48f5d2fce87892ace5). - 📣 Introducing Long-context [Reasoning (GRPO)](https://unsloth.ai/blog/grpo) in Unsloth. Train your own reasoning model with just 5GB VRAM. Transform Llama, Phi, Mistral etc. into reasoning LLMs! - 📣 Introducing Unsloth [Dynamic 4-bit Quantization](https://unsloth.ai/blog/dynamic-4bit)! We dynamically opt not to quantize certain parameters and this greatly increases accuracy while only using <10% more VRAM than BnB 4-bit. See our collection on [Hugging Face here.](https://huggingface.co/collections/unsloth/unsloth-4-bit-dynamic-quants-67503bb873f89e15276c44e7) From 2769e27566c804442e4a2c77d6a3319823b0ab6e Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Thu, 27 Nov 2025 20:52:27 -0800 Subject: [PATCH 2/7] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 04ce316f5a..490539520e 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,8 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide]( - **Docker**: Use Unsloth with no setup & environment issues with our new image. [Guide](https://docs.unsloth.ai/new/how-to-train-llms-with-unsloth-and-docker) • [Docker image](https://hub.docker.com/r/unsloth/unsloth) - **gpt-oss RL**: Introducing the fastest possible inference for gpt-oss RL! [Read blog](https://docs.unsloth.ai/new/gpt-oss-reinforcement-learning) - **Vision RL**: You can now train VLMs with GRPO or GSPO in Unsloth! [Read guide](https://docs.unsloth.ai/new/vision-reinforcement-learning-vlm-rl) -- **Quantization-Aware Training**: We collabed with Pytorch for QAT, recovering as much 70% accuracy. [Read blog](https://docs.unsloth.ai/new/quantization-aware-training-qat) -- **gpt-oss** by OpenAI: For details on [Unsloth Flex Attention](https://docs.unsloth.ai/new/long-context-gpt-oss-training), long-context training, bug fixes, [Read our Guide](https://docs.unsloth.ai/basics/gpt-oss). 20B works on a 14GB GPU and 120B on 65GB VRAM. [gpt-oss uploads](https://huggingface.co/collections/unsloth/gpt-oss-6892433695ce0dee42f31681). +- **Quantization-Aware Training**: We collabed with Pytorch, recovering ~70% accuracy. [Read blog](https://docs.unsloth.ai/new/quantization-aware-training-qat) +- **gpt-oss** by OpenAI: Read our [Unsloth Flex Attention](https://docs.unsloth.ai/new/long-context-gpt-oss-training) blog and [gpt-oss Guide](https://docs.unsloth.ai/basics/gpt-oss). 20B works on 14GB VRAM. 120B on 65GB.
Click for more news From f3803cdee031f96c25e410e78096170d7310a6b6 Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Sat, 29 Nov 2025 08:44:26 -0600 Subject: [PATCH 3/7] fix rope_theta -> rope_parameters['rope_theta'] (#3651) --- unsloth/models/_utils.py | 6 ++++++ unsloth/models/llama.py | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 6ca128bd83..cd4eeefb69 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -562,6 +562,12 @@ for model_name in model_architectures: config = inspect.getsource(eval(config_filename)) except: continue + if "RopeParameters" in config: + try: + exec(f"from {config_filepath} import RopeParameters", globals()) + except: + continue + if "rope_scaling" in config: continue config = re.sub( diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index b1b8a8fb78..f6a0fff117 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1551,7 +1551,11 @@ class LlamaRotaryEmbedding(torch.nn.Module): super().__init__() if config is not None: # [TODO] Hack to pass in config - need to remove later - base = config.rope_theta + try: + base = config.rope_theta + except: + base = getattr(config, "rope_parameters", {}) + base = base["rope_theta"] partial_rotary_factor = ( config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") From f730ad0b3eb6ec6d759c429185d2ede0a2b20c9d Mon Sep 17 00:00:00 2001 From: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> Date: Sat, 29 Nov 2025 08:01:00 -0800 Subject: [PATCH 4/7] Update README.md --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 490539520e..4ca032cf0a 100644 --- a/README.md +++ b/README.md @@ -18,18 +18,18 @@ ## ✨ Train for Free -Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then export your trained model to GGUF, Ollama, vLLM or Hugging Face. +Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-started/fine-tuning-guide). Add dataset, run, then export your trained model to GGUF, llama.cpp, Ollama, vLLM, SGLang or Hugging Face. | Model | Free Notebooks | Performance | Memory use | |-----------|---------|--------|----------| | **gpt-oss (20B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-Fine-tuning.ipynb) | 1.5x faster | 70% less | | **Qwen3 (14B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(14B)-Reasoning-Conversational.ipynb) | 2x faster | 70% less | -| **DeepSeek-OCR (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) | 1.5x faster | 30% less | | **gpt-oss (20B): GRPO** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-GRPO.ipynb) | 2x faster | 80% less | | **Qwen3-VL (8B): GSPO** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_VL_(8B)-Vision-GRPO.ipynb) | 1.5x faster | 80% less | | **Qwen3-VL (8B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_VL_(8B)-Vision.ipynb) | 2x faster | 50% less | | **Gemma 3 (270M)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Gemma3_(270M).ipynb) | 1.7x faster | 60% less | | **Gemma 3n (4B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Gemma3N_(4B)-Conversational.ipynb) | 1.5x faster | 50% less | +| **DeepSeek-OCR (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Deepseek_OCR_(3B).ipynb) | 1.5x faster | 30% less | | **Llama 3.1 (8B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3.1_(8B)-Alpaca.ipynb) | 2x faster | 70% less | | **Orpheus-TTS (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Orpheus_(3B)-TTS.ipynb) | 1.5x faster | 50% less | @@ -90,7 +90,7 @@ For RTX 50x, B200, 6000 GPUs: `pip install unsloth`. Read our [Blackwell Guide]( | ✍️ **Blog** | [Read our Blogs](https://unsloth.ai/blog)| ## ⭐ Key Features -- Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **8-bit** training +- Supports **full-finetuning**, pretraining, 4b-bit, 16-bit and **FP8** training - Supports **all models** including [TTS](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning), multimodal, [BERT](https://docs.unsloth.ai/get-started/unsloth-notebooks#other-important-notebooks) and more! Any model that works in transformers, works in Unsloth. - The most efficient library for [Reinforcement Learning (RL)](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide), using 80% less VRAM. Supports GRPO, GSPO, DrGRPO, DAPO etc. - **0% loss in accuracy** - no approximation methods - all exact. @@ -347,7 +347,7 @@ trainer = SFTTrainer( trainer.train() # Go to https://docs.unsloth.ai for advanced tips like -# (1) Saving to GGUF / merging to 16bit for vLLM +# (1) Saving to GGUF / merging to 16bit for vLLM or SGLang # (2) Continued training from a saved LoRA adapter # (3) Adding an evaluation loop / OOMs # (4) Customized chat templates @@ -355,13 +355,14 @@ trainer.train() ## 💡 Reinforcement Learning -[RL](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) including [GRPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth. +[RL](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) including [GRPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide#training-with-grpo), [GSPO](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/gspo-reinforcement-learning), **FP8** traning, DrGRPO, DAPO, PPO, Reward Modelling, Online DPO all work with Unsloth. Read our [Reinforcement Learning Guide](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide) or our [advanced RL docs](https://docs.unsloth.ai/get-started/reinforcement-learning-rl-guide/advanced-rl-documentation) for batching, generation & training parameters. List of RL notebooks: - gpt-oss GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/gpt-oss-(20B)-GRPO.ipynb) - Qwen2.5-VL GSPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen2_5_7B_VL_GRPO.ipynb) - Advanced Qwen3 GRPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb) +- ***FP8*** Qwen3-8B GRPO notebook (L4): [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_8B_FP8_GRPO.ipynb) - ORPO notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Llama3_(8B)-ORPO.ipynb) - DPO Zephyr notebook: [Link](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Zephyr_(7B)-DPO.ipynb) - KTO notebook: [Link](https://colab.research.google.com/drive/1MRgGtLWuZX4ypSfGguFgC-IblTvO2ivM?usp=sharing) From d8ea26192411df6d2856197196e89fa826bdf692 Mon Sep 17 00:00:00 2001 From: gitpullpull <147926812+gitpullpull@users.noreply.github.com> Date: Sun, 30 Nov 2025 08:33:48 +0900 Subject: [PATCH 5/7] Fix broken link for Advanced pip install instructions (#3652) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4ca032cf0a..37c1c89ede 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ pip install unsloth ``` pip install --upgrade --force-reinstall --no-cache-dir unsloth unsloth_zoo ``` -See [here](https://github.com/unslothai/unsloth/edit/main/README.md#advanced-pip-installation) for advanced pip install instructions. +See [here](#advanced-pip-installation) for advanced pip install instructions. ### Windows Installation 1. **Install NVIDIA Video Driver:** From 27ae5c335cba438e6198c4fdbabe9a0a85b329b4 Mon Sep 17 00:00:00 2001 From: Bhuvan Prakash <52481324+bhuvanprakash@users.noreply.github.com> Date: Sun, 30 Nov 2025 09:48:11 +0530 Subject: [PATCH 6/7] Fix: prevent load_in_fp8 kwarg from reaching Qwen3MoeForCausalLM constructor (Fix #3649) (#3654) * Fix: remove load_in_fp8 from kwargs to prevent Qwen3Moe init TypeError (Fix #3649) * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- unsloth/models/vision.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 30d381c791..31901b1570 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -654,6 +654,8 @@ class FastBaseModel: raise_handler = RaiseUninitialized() if not fast_inference: + # Prevent load_in_fp8 from being forwarded into HF internal model loading + load_in_fp8 = kwargs.pop("load_in_fp8", None) model = auto_model.from_pretrained( model_name, device_map = device_map, From 69f7dfdb901a72f99d9d019fab2ddba3135a3efa Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Sun, 30 Nov 2025 00:47:51 -0600 Subject: [PATCH 7/7] make unsloth_tiled_mlp a from_pretrained arg (#3655) * make unsloth_tiled_mlp a from_pretrained arg * adjust patching logic --- unsloth/models/loader.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index a8ddbfed2a..e1c13315f7 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -146,6 +146,7 @@ class FastLanguageModel(FastLlamaModel): disable_log_stats = True, qat_scheme = None, load_in_fp8 = False, # fp8 LoRA (True, False, 'block') + unsloth_tiled_mlp = False, *args, **kwargs, ): @@ -512,6 +513,7 @@ class FastLanguageModel(FastLlamaModel): disable_log_stats = disable_log_stats, qat_scheme = qat_scheme, load_in_fp8 = load_in_fp8, + unsloth_tiled_mlp = unsloth_tiled_mlp, *args, **kwargs, ) @@ -609,8 +611,10 @@ class FastLanguageModel(FastLlamaModel): # Patch Tiled MLP # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}"" - patch_tiled_mlp_choice = os.environ.get("UNSLOTH_TILED_MLP", "0") - if patch_tiled_mlp_choice != "0": + patch_tiled_mlp_choice = os.environ.get( + "UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0" + ) + if patch_tiled_mlp_choice != "0" or unsloth_tiled_mlp: patch_tiled_mlp(model, patch_options_str = patch_tiled_mlp_choice) return model, tokenizer @@ -674,6 +678,7 @@ class FastModel(FastBaseModel): disable_log_stats = True, qat_scheme = None, load_in_fp8 = False, # fp8 LoRA (True, False, 'block') + unsloth_tiled_mlp = False, *args, **kwargs, ): @@ -1240,8 +1245,10 @@ class FastModel(FastBaseModel): # Patch Tiled MLP # to turn on set UNSLOTH_TILED_MLP to "arctic", "target", or "target:{GB}"" - patch_tiled_mlp_choice = os.environ.get("UNSLOTH_TILED_MLP", "0") - if patch_tiled_mlp_choice != "0": + patch_tiled_mlp_choice = os.environ.get( + "UNSLOTH_TILED_MLP", "arctic" if unsloth_tiled_mlp else "0" + ) + if patch_tiled_mlp_choice != "0" or unsloth_tiled_mlp: patch_tiled_mlp(model, patch_options_str = patch_tiled_mlp_choice) return model, tokenizer