diff --git a/README.md b/README.md index 6279117888..24850bb974 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ -### Finetune Qwen3, Llama 4, Gemma 3, Phi-4 & Mistral 2x faster with 80% less VRAM! +### Finetune Gemma 3n, Qwen3, Llama 4, Phi-4 & Mistral 2x faster with 80% less VRAM! ![](https://i.ibb.co/sJ7RhGG/image-41.png) @@ -22,6 +22,7 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st | Unsloth supports | Free Notebooks | Performance | Memory use | |-----------|---------|--------|----------| +| **Gemma 3n (4B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Gemma3N_(4B)-Conversational.ipynb) | 1.5x faster | 50% less | | **Qwen3 (14B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(14B)-Reasoning-Conversational.ipynb) | 2x faster | 70% less | | **Qwen3 (4B): GRPO** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Qwen3_(4B)-GRPO.ipynb) | 2x faster | 80% less | | **Gemma 3 (4B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Gemma3_(4B).ipynb) | 1.6x faster | 60% less | @@ -45,12 +46,12 @@ pip install unsloth For Windows install instructions, see [here](https://docs.unsloth.ai/get-started/installing-+-updating/windows-installation). ## 🦥 Unsloth.ai News -- 📣 NEW! **[Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`. -- 📣 NEW! **[Qwen3](https://docs.unsloth.ai/basics/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM. -- 📣 NEW! Introducing **[Dynamic 2.0](https://docs.unsloth.ai/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & KL Divergence. +- 📣 **Gemma 3n** by Google: [Read Blog](https://docs.unsloth.ai/basics/gemma-3n-how-to-run-and-fine-tune). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3n-685d3874830e49e1c93f9339). +- 📣 **[Text-to-Speech (TTS)](https://docs.unsloth.ai/basics/text-to-speech-tts-fine-tuning)** is now supported, including `sesame/csm-1b` and STT `openai/whisper-large-v3`. +- 📣 **[Qwen3](https://docs.unsloth.ai/basics/qwen3-how-to-run-and-fine-tune)** is now supported. Qwen3-30B-A3B fits on 17.5GB VRAM. +- 📣 Introducing **[Dynamic 2.0](https://docs.unsloth.ai/basics/unsloth-dynamic-2.0-ggufs)** quants that set new benchmarks on 5-shot MMLU & KL Divergence. - 📣 **[Llama 4](https://unsloth.ai/blog/llama4)** by Meta, including Scout & Maverick are now supported. - 📣 [**EVERYTHING** is now supported](https://unsloth.ai/blog/gemma3#everything) - all models (BERT, diffusion, Cohere, Mamba), FFT, etc. MultiGPU coming soon. Enable FFT with `full_finetuning = True`, 8-bit with `load_in_8bit = True`. -- 📣 **Gemma 3** by Google: [Read Blog](https://unsloth.ai/blog/gemma3). We [uploaded GGUFs, 4-bit models](https://huggingface.co/collections/unsloth/gemma-3-67d12b7e8816ec6efa7e4e5b). - 📣 Introducing Long-context [Reasoning (GRPO)](https://unsloth.ai/blog/grpo) in Unsloth. Train your own reasoning model with just 5GB VRAM. Transform Llama, Phi, Mistral etc. into reasoning LLMs! - 📣 [DeepSeek-R1](https://unsloth.ai/blog/deepseek-r1) - run or fine-tune them [with our guide](https://unsloth.ai/blog/deepseek-r1). All model uploads: [here](https://huggingface.co/collections/unsloth/deepseek-r1-all-versions-678e1c48f5d2fce87892ace5).
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 2e88f7675e..5da6ea67fe 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -206,33 +206,18 @@ except: # Patch get_model_param_count to record correct 4bit / 8bit from transformers.trainer_pt_utils import is_deepspeed_zero3_enabled -def extract_approx_params_from_config(config): +def extract_quant_model_param_count(model): """ - Extract approximate parameter count from model config's name_or_path - Returns int (param count) or None if not found. + Calculate quant model param count based on difference in param class. Returns int for param count. """ - lowercase_b_families = ["gemma"] # gemma uses small 'b' : google/gemma-3-1b-it - model_name = getattr(config, "name_or_path", "") - import re - cleaned = re.sub(r"[-_]?bnb[-_]?4bit|[-_]?4bit|[-_]?8bit|[-_]?bnb", "", model_name, flags=re.IGNORECASE) # replace bnb and xbit - match_B = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*B", cleaned) # first prefer searching 'B' - if match_B: - # most model names would come in this flow - billions = float(match_B.group(1)) - return int(1_000_000_000 * billions) - else: - if any(fam in cleaned.lower() for fam in lowercase_b_families): - match_b = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*b", cleaned) - if match_b: - billions = float(match_b.group(1)) - return int(1_000_000_000 * billions) + count: int = 0 + for name, p in model.named_parameters(): + if p.__class__.__name__ == "Params4bit": + count += 2 * p.numel() else: - match_any = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*[bB]", cleaned) - if match_any: - billions = float(match_any.group(1)) - return int(1_000_000_000 * billions) - return None - + count += p.numel() + return count +pass def get_model_param_count(model, trainable_only = False): """ @@ -248,7 +233,7 @@ def get_model_param_count(model, trainable_only = False): if (not trainable_only) and \ hasattr(model, "config") and \ hasattr(model.config, "quantization_config"): - approx = extract_approx_params_from_config(model.config) + approx = extract_quant_model_param_count(model) if approx is not None: s = approx return s @@ -370,7 +355,7 @@ if is_openai_available(): def _is_openai_available(): return False transformers.utils.is_openai_available = _is_openai_available pass -pass +pass # ============================================= # Get Flash Attention v2 if Ampere (RTX 30xx, A100) @@ -1085,7 +1070,7 @@ pass def patch_gradient_accumulation_fix(Trainer): - # Fixes gradient accumulation + # Fixes gradient accumulation import inspect if hasattr(Trainer, "get_batch_samples"): if Trainer.get_batch_samples.__name__ == "_unsloth_get_batch_samples": return @@ -1159,10 +1144,10 @@ def patch_gradient_accumulation_fix(Trainer): "\2if num_items_in_batch is None:\n"\ "\3loss = loss / self.args.gradient_accumulation_steps\n"\ "\1self.accelerator.backward(loss, **kwargs)", - + function, ) - + exec(function, globals()) Trainer.training_step = _unsloth_training_step pass @@ -1356,7 +1341,7 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m ) loftq_config = LoftQConfig(loftq_bits = 4, loftq_iter = 1) pass - + if hasattr(model.config, "quantization_config"): raise ValueError( "Unsloth: You are using `loftq` init, yet `load_in_4bit = True` was set.\n"\ @@ -1365,4 +1350,4 @@ def validate_loftq_config(loftq_config, lora_dropout, bias, init_lora_weights, m pass pass - return loftq_config \ No newline at end of file + return loftq_config diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index a95a54b59d..683f2b1872 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -561,6 +561,7 @@ class FastModel(FastBaseModel): raise RuntimeError("Unsloth: Cohere's Command model only works on transformers >= 4.50.0." + NIGHTLY) # Sesame elif "csm-1b" in lowered_model_name: + os.environ["UNSLOTH_COMPILE_DISABLE"] = "1" # Inference is too slow os.environ["UNSLOTH_DISABLE_STATIC_GENERATION"] = "1" # Sesame fails os.environ["UNSLOTH_FORCE_CUSTOM_DTYPE"] = \ "all;torch.float32;torch.float16;"\