diff --git a/README.md b/README.md index 4b5f0cb480..1cb5fe9bee 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Notebooks are beginner friendly. Read our [guide](https://docs.unsloth.ai/get-st | **Mistral v0.3 (7B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Mistral_v0.3_(7B)-Conversational.ipynb) | 2.2x faster | 75% less | | **Orpheus-TTS (3B)** | [▶️ Start for free](https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/Orpheus_(3B)-TTS.ipynb) | 1.5x faster | 50% less | -- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://docs.unsloth.ai/get-started/unsloth-notebooks), **[TTS](https://docs.unsloth.ai/get-started/unsloth-notebooks#text-to-speech-tts-notebooks)** & [Vision](https://docs.unsloth.ai/get-started/unsloth-notebooks#vision-multimodal-notebooks) +- See all our notebooks for: [Kaggle](https://github.com/unslothai/notebooks?tab=readme-ov-file#-kaggle-notebooks), [GRPO](https://docs.unsloth.ai/get-started/unsloth-notebooks#grpo-reasoning-rl-notebooks), **[TTS](https://docs.unsloth.ai/get-started/unsloth-notebooks#text-to-speech-tts-notebooks)** & [Vision](https://docs.unsloth.ai/get-started/unsloth-notebooks#vision-multimodal-notebooks) - See [all our models](https://docs.unsloth.ai/get-started/all-our-models) and [all our notebooks](https://github.com/unslothai/notebooks) - See detailed documentation for Unsloth [here](https://docs.unsloth.ai/) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index c2520a66b5..5b3879ce68 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -66,6 +66,7 @@ __all__ = [ "unsloth_compile_transformers", "patch_fast_lora", "validate_loftq_config", + "RaiseUninitialized", ] import torch @@ -205,6 +206,28 @@ try: except: pass +# Errors out on +# Some weights of Gemma3nForConditionalGeneration were not initialized from the model checkpoint +from transformers.modeling_utils import logger as transformers_logger +class _RaiseUninitialized(logging.Handler): + def __init__(self): + super().__init__() + def emit(self, record): + if "some weights of" in str(record).lower(): + raise Exception( + f"Unsloth: Critical error since some weights are not initialized.\n"\ + f"Please try updating Unsloth, transformers and timm via:\n"\ + f"`pip install --upgrade --force-reinstall --no-cache-dir --no-deps unsloth unsloth_zoo transformers timm`\n"\ + f"".str(record)) +pass +class RaiseUninitialized: + def __init__(self): + self.error_handler = _RaiseUninitialized() + transformers_logger.addHandler(self.error_handler) + def remove(self): + transformers_logger.removeHandler(self.error_handler) +pass + # Patch get_model_param_count to record correct 4bit / 8bit from transformers.trainer_pt_utils import is_deepspeed_zero3_enabled diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index f24b6b7e17..44bd08c028 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1967,6 +1967,7 @@ class FastLlamaModel: # Cannot be None, since HF now checks for the config if load_in_4bit: kwargs["quantization_config"] = bnb_config + raise_handler = RaiseUninitialized() if num_labels is not None: model = AutoModelForSequenceClassification.from_pretrained( model_name, @@ -2030,6 +2031,7 @@ class FastLlamaModel: model.fast_generate = model.vllm_engine.generate model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine) pass + raise_handler.remove() # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer @@ -2540,11 +2542,8 @@ class FastLlamaModel: raise NotImplementedError("Unsloth: Currently fast inference does not work with using biases for LoRA.") pass - #does not get lora yet, so get name from model, not base model - - is_classification = "Classification" in str(type(model)) - # Get LoRA - # + #d oes not get lora yet, so get name from model, not base model + is_classification = "Classification" in str(type(model)) arguments = dict( r = r, @@ -2696,16 +2695,16 @@ class FastLlamaModel: # Get activation function model_type = model.config.model_type - if model_type == "llama": apply_lora_mlp = apply_lora_mlp_swiglu - elif model_type == "mistral": apply_lora_mlp = apply_lora_mlp_swiglu - elif model_type == "qwen2": apply_lora_mlp = apply_lora_mlp_swiglu - elif model_type == "gemma": apply_lora_mlp = apply_lora_mlp_geglu_approx - elif model_type == "gemma2": apply_lora_mlp = apply_lora_mlp_geglu_approx - elif model_type == "cohere": apply_lora_mlp = apply_lora_mlp_swiglu - elif model_type == "granite": apply_lora_mlp = apply_lora_mlp_swiglu - elif model_type == "qwen3": apply_lora_mlp = apply_lora_mlp_swiglu - elif model_type == "falcon_h1": apply_lora_mlp = apply_lora_mlp_swiglu - elif model_type == "qwen3moe": apply_lora_mlp = apply_lora_mlp_swiglu + if model_type == "llama": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "mistral": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "qwen2": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "gemma": apply_lora_mlp = apply_lora_mlp_geglu_approx + elif model_type == "gemma2": apply_lora_mlp = apply_lora_mlp_geglu_approx + elif model_type == "cohere": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "granite": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "qwen3": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "falcon_h1": apply_lora_mlp = apply_lora_mlp_swiglu + elif model_type == "qwen3moe": apply_lora_mlp = apply_lora_mlp_swiglu else: raise NotImplementedError(f"Unsloth: {model_type} is not yet implemented!") pass diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 889bbd4807..ae01469acc 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -458,26 +458,33 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): # Edit GA / bsz and weight_decay replacements = { - "output_dir" : None, - "logging_nan_inf_filter" : False, - "per_device_train_batch_size" : 4, - "gradient_accumulation_steps" : 2, - "weight_decay" : 0.01, - "warmup_ratio" : 0.1, - "seed" : 3407, - "optim" : "adamw_8bit", - "learning_rate" : 5e-05, - "per_device_eval_batch_size" : 4, - "eval_accumulation_steps" : 2, - "torch_empty_cache_steps" : 250, - "logging_steps" : 1, - "max_seq_length" : None, - "num_generations" : 8, - "top_k" : None, - "vllm_mode" : "colocate", - "generation_kwargs" : {}, - "bf16" : False, - "fp16" : False, + "output_dir" : None, + "logging_nan_inf_filter" : False, + "per_device_train_batch_size" : 4, + "gradient_accumulation_steps" : 2, + "weight_decay" : 0.01, + "warmup_ratio" : 0.1, + "seed" : 3407, + "optim" : "adamw_8bit", + "learning_rate" : 5e-05, + "per_device_eval_batch_size" : 4, + "eval_accumulation_steps" : 2, + "torch_empty_cache_steps" : 250, + "logging_steps" : 1, + "max_seq_length" : None, + "num_generations" : 8, + "top_k" : None, + "vllm_mode" : "colocate", + "generation_kwargs" : {}, + "bf16" : False, + "fp16" : False, + "include_tokens_per_second" : False, + "include_num_input_tokens_seen" : False, + "auto_find_batch_size" : True, # Auto /2 batch size + "dataloader_persistent_workers" : True, # Keeps dataloader in RAM + "dataloader_prefetch_factor" : 2, + "dataloader_pin_memory" : True, + "dataloader_num_workers" : 0, # Default is 0 means 1 } for k, v in replacements.items(): x = f"{k}( = [^,\n]{{1,}})?,\n" @@ -526,7 +533,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): num_proc_check = \ "if dataset_num_proc is None:\n"\ " from multiprocessing import cpu_count\n"\ - " dataset_num_proc = cpu_count()\n" + " dataset_num_proc = min(cpu_count()*2, 2)\n" extra_args += num_proc_check pass diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index a358594d85..9899d60440 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -420,6 +420,7 @@ class FastBaseModel: torch_dtype = dtype if do_forced_float32: torch_dtype = torch.bfloat16 + raise_handler = RaiseUninitialized() model = auto_model.from_pretrained( model_name, device_map = device_map, @@ -430,6 +431,7 @@ class FastBaseModel: # attn_implementation = attn_implementation, **kwargs, ) + raise_handler.remove() # Return old flag os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer