From 1b28ec31f715becaac58b9e6e1296241201d1bcd Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 17 Jul 2025 07:02:38 -0700 Subject: [PATCH 1/5] Bug fixes (#2982) * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Synthetic data * Update mapper.py * Xet and Synthetic * Update synthetic.py * Update loader.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update chat_templates.py * Seasame force float16 / float32 * Fix Seasame * Update loader.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * is_multimodal * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update vision.py * Update vision.py * Update vision.py * UNSLOTH_DISABLE_STATIC_GENERATION * Update vision.py * Auto vision detection * Sesame * Whisper * Update loader.py * Update loader.py * Update loader.py * Update mapper.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Update loader.py * Update loader.py * Update _utils.py * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl.py * Update rl.py * Update rl.py * logging * Update pyproject.toml * Update rl.py * versioning * Update rl.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * logits / temperature * Update rl_replacements.py * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Debugging only * Update llama.py * Update llama.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Generic efficient GRPO * Update rl_replacements.py * Update rl_replacements.py * Remove debugging * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update llama.py * Update rl_replacements.py * versioning * Update _utils.py * Update vision.py * Update mapper.py * Update loader.py * Update mapper.py * Update vision.py * Update loader.py * Update vision.py * Update loader.py * Update _utils.py * Update vision.py * gradient checkpointing * Gemma 3N fixes * Update loader.py * Versioning * Gemma 3N fixes * Update vision.py * Update vision.py * Update loader.py * Update vision.py * Fix setup.py * setup.py * Prints * Update setup.py * Update setup.py * Update setup.py * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update vision.py * Update vision.py * Update pyproject.toml * Update vision.py * Update _utils.py * Update __init__.py * Update __init__.py * Small fixes * Update vision.py * Update vision.py * versioning * Update __init__.py * Update llama.py * Update rl.py * Update rl.py * Update _utils.py * Update vision.py * Update vision.py * compiler stance * Update _utils.py * Update pyproject.toml * Update pyproject.toml * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py --- pyproject.toml | 8 ++--- unsloth/models/_utils.py | 10 +++++- unsloth/models/llama.py | 4 +-- unsloth/models/rl.py | 9 +++--- unsloth/models/rl_replacements.py | 53 +++++++++++++++++++++++++++++++ unsloth/models/vision.py | 4 +++ 6 files changed, 77 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ae46d7aaf5..d17859cfa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.7.4", + "unsloth_zoo>=2025.7.5", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -48,7 +48,7 @@ huggingface = [ "wheel>=0.42.0", "numpy", "accelerate>=0.34.1", - "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", "peft>=0.7.1,!=0.11.0", "protobuf", "huggingface_hub", @@ -381,7 +381,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.7.4", + "unsloth_zoo>=2025.7.5", "packaging", "tyro", "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0", @@ -399,7 +399,7 @@ colab-new = [ ] colab-no-deps = [ "accelerate>=0.34.1", - "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", "peft>=0.7.1", "xformers", "bitsandbytes>=0.45.5", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 5b3879ce68..bfd7c8c5a5 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.7.3" +__version__ = "2025.7.4" __all__ = [ "SUPPORTS_BFLOAT16", @@ -190,6 +190,14 @@ try: except: pass +# The following generation flags are not valid and may be ignored: +try: + from transformers.generation.configuration_utils import logger as configuration_logger + configuration_logger.addFilter(HideLoggingMessage("following generation flags")) + del configuration_logger +except: + pass + # Gemma3 It is strongly recommended to train Gemma3 models with the `eager` try: from transformers.models.gemma3.modeling_gemma3 import logger as gemma3_logger diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index 3551c6ca5e..8d985aa9d2 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -2224,7 +2224,7 @@ class FastLlamaModel: bias = "none", layers_to_transform = None, layers_pattern = None, - use_gradient_checkpointing = True, + use_gradient_checkpointing = "unsloth", random_state = 3407, max_seq_length = 2048, # not used anymore use_rslora = False, @@ -2678,7 +2678,7 @@ class FastLlamaModel: @staticmethod def patch_peft_model( model, - use_gradient_checkpointing = True, + use_gradient_checkpointing = "unsloth", ): if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1": return FastBaseModel.patch_peft_model( diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py index 45b8ca6334..664fe10c4f 100644 --- a/unsloth/models/rl.py +++ b/unsloth/models/rl.py @@ -481,10 +481,11 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): "include_tokens_per_second" : False, "include_num_input_tokens_seen" : False, "auto_find_batch_size" : True, # Auto /2 batch size - "dataloader_persistent_workers" : True, # Keeps dataloader in RAM - "dataloader_prefetch_factor" : 2, "dataloader_pin_memory" : True, - "dataloader_num_workers" : 1, + # Might fail so disable for now + # "dataloader_persistent_workers" : True, # Keeps dataloader in RAM + # "dataloader_prefetch_factor" : 2, + # "dataloader_num_workers" : 2, # Default is 0 means 1 } for k, v in replacements.items(): x = f"{k}( = [^,\n]{{1,}})?,\n" @@ -671,7 +672,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"): RLTrainer_source, f"trl.trainer.{trainer_file}", imports, - overwrite = False, + overwrite = True, ) # Patch Trainer diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index 4d5b4d7a4d..a88385bf03 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -235,6 +235,59 @@ pass RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__prepare_inputs) +# Fix incorrect special tokens handling and truncation in older TRL versions +def grpo_trainer__generate_and_score_completions(function_name, function): + if function_name != "_generate_and_score_completions": return function + + # TRL 0.19.0 did skip_special_tokens = True which should be False + function = function.replace( + "prompt_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False", + "prompt_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False", + ) + + # Always between max_prompt_length and use_vllm + found = re.findall( + r"\n(([ ]{8,})if self\.max_prompt_length is not None:.*?"\ + r"\2if self\.use_vllm:)", + function, + flags = re.DOTALL | re.MULTILINE, + ) + if len(found) != 0: + replace_part, spacing = found[0] + removed_comments = re.sub(r"\#[^\n]{1,}", "", replace_part) + splits = removed_comments.split("\n") + if sum(re.match(rf"{spacing}[^\s]", x) is not None for x in splits) == 2 and len(spacing) >= 8: + + new_replacement = \ + f"""\n{spacing}if self.max_prompt_length is not None: + # If max_prompt_length is set, we trim the prompt to keep only the last `max_prompt_length` tokens. + # Then we decode those tokens back into text. We manually remove leading pad tokens from the decoded text, + # because we can't use `skip_special_tokens=True` (some special tokens are still needed for generation). + prompt_ids = prompt_ids[:, -self.max_prompt_length :] + prompt_mask = prompt_mask[:, -self.max_prompt_length :] + prompts_text = self.processing_class.batch_decode( + prompt_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False + ) + pad_token = self.processing_class.pad_token + def strip_leading_tokens(text): + while text.startswith(pad_token): + text = text.removeprefix(pad_token) + return text + + if pad_token is not None: + prompts_text = [ + strip_leading_tokens(text) for text in prompts_text + ] + + # Generate completions using either vLLM or regular generation + if self.use_vllm:""" + function = function.replace(replace_part, new_replacement) + pass + return function +pass +RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__generate_and_score_completions) + + # Remove _move_model_to_vllm def grpo_trainer__move_model_to_vllm(function_name, function): if function_name != "_move_model_to_vllm": return function diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 7442f07e73..5bbf4c76a6 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -755,6 +755,8 @@ class FastBaseModel: os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0" # Must enable returning logits os.environ["UNSLOTH_RETURN_LOGITS"] = "1" + # Turn off skip guards and set stance to default + torch.compiler.set_stance(stance = "default", skip_guard_eval_unsafe = False) return model pass @@ -801,6 +803,8 @@ class FastBaseModel: pass # Can re-enable not returning logits os.environ["UNSLOTH_RETURN_LOGITS"] = "0" + # Turn off skip guards and set stance to default + torch.compiler.set_stance(stance = "default", skip_guard_eval_unsafe = False) return model pass pass From be955324c4cc73825e402ec4c1d219ac2e31baf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Thu, 17 Jul 2025 15:08:38 -0700 Subject: [PATCH 2/5] Update unsloth-cli.py (#2985) --- unsloth-cli.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/unsloth-cli.py b/unsloth-cli.py index 86f02075f1..a782cfd743 100644 --- a/unsloth-cli.py +++ b/unsloth-cli.py @@ -38,7 +38,7 @@ def run(args): from unsloth import FastLanguageModel from datasets import load_dataset from transformers.utils import strtobool - from trl import SFTTrainer + from trl import SFTTrainer, SFTConfig from transformers import TrainingArguments from unsloth import is_bfloat16_supported import logging @@ -100,7 +100,7 @@ def run(args): print("Data is formatted and ready!") # Configure training arguments - training_args = TrainingArguments( + training_args = SFTConfig( per_device_train_batch_size=args.per_device_train_batch_size, gradient_accumulation_steps=args.gradient_accumulation_steps, warmup_steps=args.warmup_steps, @@ -115,17 +115,16 @@ def run(args): seed=args.seed, output_dir=args.output_dir, report_to=args.report_to, + max_length=args.max_seq_length, + dataset_num_proc=2, + packing=False, ) # Initialize trainer trainer = SFTTrainer( model=model, - tokenizer=tokenizer, + processing_class=tokenizer, train_dataset=dataset, - dataset_text_field="text", - max_seq_length=args.max_seq_length, - dataset_num_proc=2, - packing=False, args=training_args, ) From 8362af247593244b6bd3578a7b6db01a5d7e38c1 Mon Sep 17 00:00:00 2001 From: DoubleMathew Date: Thu, 17 Jul 2025 17:34:14 -0500 Subject: [PATCH 3/5] use fastmodel (#2987) --- unsloth/models/loader.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 683f2b1872..953991d41d 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -320,15 +320,15 @@ class FastLanguageModel(FastLlamaModel): f"to obtain the latest transformers build, then restart this session."\ ) dispatch_model = FastQwen3Model if model_type == "qwen3" else FastQwen3MoeModel - elif model_type == "falcon_h1": - dispatch_model = FastFalconH1Model - if not SUPPORTS_FALCON_H1: - raise ImportError( - f"Unsloth: Your transformers version of {transformers_version} does not support FalconH1.\n"\ - f"The minimum required version is 4.50.3.\n"\ - f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\ - f"to obtain the latest transformers build, then restart this session."\ - ) + # elif model_type == "falcon_h1": + # dispatch_model = FastFalconH1Model + # if not SUPPORTS_FALCON_H1: + # raise ImportError( + # f"Unsloth: Your transformers version of {transformers_version} does not support FalconH1.\n"\ + # f"The minimum required version is 4.50.3.\n"\ + # f'Try `pip install --upgrade "transformers>=4.50.3"`\n'\ + # f"to obtain the latest transformers build, then restart this session."\ + # ) # Temporary disable optimized Cohere until errors match # elif model_type == "cohere": # dispatch_model = FastCohereModel From 73c02231dee188180d6678d24a4017322eea416f Mon Sep 17 00:00:00 2001 From: Roland Tannous <115670425+rolandtannous@users.noreply.github.com> Date: Fri, 18 Jul 2025 01:35:05 +0300 Subject: [PATCH 4/5] Add Qwen2.5-VL-32B-Instruct mapping to fix quantized model merge error (#2986) --- ..._merge_qwen2.5vl32B_model_ocr_benchmark.py | 255 ++++++++++++++++++ unsloth/models/mapper.py | 5 + 2 files changed, 260 insertions(+) create mode 100644 tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py diff --git a/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py b/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py new file mode 100644 index 0000000000..0bf548b41c --- /dev/null +++ b/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py @@ -0,0 +1,255 @@ +# -*- coding: utf-8 -*- + +from unsloth import FastVisionModel + +import torch +from qwen_vl_utils import process_vision_info +import os +from datasets import load_dataset +from trl import SFTTrainer, SFTConfig + +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).parents[3] +sys.path.insert(0, str(REPO_ROOT)) + +from tests.utils.cleanup_utils import safe_remove_directory +from tests.utils.ocr_eval import OCRModelEvaluator + + +## Dataset Preparation +from datasets import load_dataset + +dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", 'en', split="train") +# To select the first 2000 examples +train_dataset = dataset.select(range(2000)) + +# To select the next 200 examples for evaluation +eval_dataset = dataset.select(range(2000, 2200)) + +# Convert dataset to OAI messages +def format_data(sample): + return {"messages": [ + { + "role": "system", + "content": [{"type": "text", "text": system_message}], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": sample["question"], + },{ + "type": "image", + "image": sample["image"], + } + ], + }, + { + "role": "assistant", + "content": [{"type": "text", "text": sample["answer"]}], + }, + ], + } + +system_message = "You are an expert french ocr system." +# Convert dataset to OAI messages +# need to use list comprehension to keep Pil.Image type, .mape convert image to bytes +train_dataset = [format_data(sample) for sample in train_dataset] +eval_dataset = [format_data(sample) for sample in eval_dataset] + +## Setup OCR main evaluation function and helpers +import os +import torch +from tqdm import tqdm +import pandas as pd +from jiwer import wer, cer +from qwen_vl_utils import process_vision_info + +# +ocr_evaluator = OCRModelEvaluator() +model_comparison_results = {} + +## Finetuning Setup and Run +# Load Base Model + +model, tokenizer = FastVisionModel.from_pretrained( + model_name = "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit", + max_seq_length = 2048, # Choose any for long context! + load_in_4bit = True, # 4 bit quantization to reduce memory + load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory + full_finetuning = False, # [NEW!] We have full finetuning now! +) + +# benchmark base model performance +model_name = "Unsloth Base model" +FastVisionModel.for_inference(model) +avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_base_model_results") +ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) + +## Lora Finetuning +model = FastVisionModel.get_peft_model( + model, + finetune_vision_layers = True, # Turn off for just text! + finetune_language_layers = True, # Should leave on! + finetune_attention_modules = True, # Attention good for GRPO + finetune_mlp_modules = True, # SHould leave on always! + + r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 + #target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", + #"gate_proj", "up_proj", "down_proj",], + lora_alpha = 32, + lora_dropout = 0, # Supports any, but = 0 is optimized + bias = "none", # Supports any, but = "none" is optimized + # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes! + use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context + random_state = 3407, + use_rslora = False, # We support rank stabilized LoRA + loftq_config = None, # And LoftQ +) + +from unsloth import is_bf16_supported +from unsloth.trainer import UnslothVisionDataCollator +FastVisionModel.for_training(model) # Enable for training! +model.config.use_cache = False + + +trainer = SFTTrainer( + model = model, + tokenizer = tokenizer, + data_collator = UnslothVisionDataCollator(model, tokenizer), + train_dataset = train_dataset, + args = SFTConfig( + #per_device_train_batch_size = 4, + #gradient_accumulation_steps = 8, + per_device_train_batch_size = 2, + gradient_accumulation_steps = 4, + gradient_checkpointing=True, + gradient_checkpointing_kwargs = {"use_reentrant": False}, # use reentrant checkpointing + max_grad_norm=0.3, # max gradient norm based on QLoRA paper + warmup_ratio=0.03, + #num_train_epochs = 2, # Set this instead of max_steps for full training runs + max_steps=60, + learning_rate = 2e-4, + fp16 = not is_bf16_supported(), + bf16 = is_bf16_supported(), + logging_steps = 5, + save_strategy="epoch", + optim = "adamw_torch_fused", + weight_decay = 0.01, + lr_scheduler_type = "linear", + seed = 3407, + output_dir = "unsloth-qwen2.5-vl-32b-french-ocr-checkpoints", + report_to = "none", # For Weights and Biases + + # You MUST put the below items for vision finetuning: + remove_unused_columns = False, + dataset_text_field = "", + dataset_kwargs = {"skip_prepare_dataset": True}, + dataset_num_proc = 4, + max_seq_length = 2048, + ), +) + +# run training +trainer_stats = trainer.train() + +model.save_pretrained("unsloth-qwen2.5-vl-32b-french-ocr-adapter", tokenizer) +tokenizer.save_pretrained("unsloth-qwen2.5-vl-32b-french-ocr-adapter") + +## Measure Adapter Performance + +# benchmark lora model performance +model_name = "Unsloth lora adapter model" +FastVisionModel.for_inference(model) +avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_lora_model_results") +ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) + +## Merge Model + +def find_lora_base_model(model_to_inspect): + current = model_to_inspect + if hasattr(current, "base_model"): + current = current.base_model + if hasattr(current, "model"): + current = current.model + return current +pass + +base = find_lora_base_model(model) + +print((base.__class__.__name__)) + +# merge default 16 bits +model.save_pretrained_merged(save_directory="qwen2.5-ocr-merged-finetune-merge-16bit", tokenizer=tokenizer) + + +## Benchmark merged model performance + +### 16 bits merged model + +model, tokenizer = FastVisionModel.from_pretrained("./qwen2.5-ocr-merged-finetune-merge-16bit",load_in_4bit=False, load_in_8bit=False) + +# benchmark 4bit loaded, 16bits merged model performance +model_name = "Unsloth 16bits-merged model load-16bits" +model.config.use_cache = True + +avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_16bits_merged_model_load_16bits_results") +ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) + +# load 16bits-merged model in 4 bits +model, tokenizer = FastVisionModel.from_pretrained("./qwen2.5-ocr-merged-finetune-merge-16bit",load_in_4bit=True, load_in_8bit=False) + +# benchmark 4bit loaded, 16bits merged model performance +model_name = "Unsloth 16bits-merged model load-4bits" +model.config.use_cache = True + +avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_16bits_merged_model_load_4bits_results") +ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) + +# load model in 8 bits +model, tokenizer = FastVisionModel.from_pretrained("./qwen2.5-ocr-merged-finetune-merge-16bit",load_in_4bit=False, load_in_8bit=True) + +# benchmark 4bit loaded, 16bits merged model performance +model_name = "Unsloth 16bits-merged model load-8bits" +avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_16bits_merged_model_load_8bits_results") +ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) + +# """### 4 bits merged model""" +# +# # load 4bits-merged model in 4 bits +# model, tokenizer = FastVisionModel.from_pretrained("./qwen2-ocr-merged-finetune-merge-4bit",load_in_4bit=True, load_in_8bit=False) +# +# # benchmark 4bit loaded, 4bits merged model performance +# model_name = "Unsloth 4bits-merged model load-4bits" +# +# avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_4bits_merged_model_load_4bits_results") +# ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) +# +# # load model in 8 bits +# model, tokenizer = FastVisionModel.from_pretrained("./qwen2-ocr-merged-finetune-merge-4bit",load_in_4bit=False, load_in_8bit=True) +# +# # benchmark 8bit loaded, 4bits merged model performance +# model_name = "Unsloth 4bits-merged model load-8bits" +# +# avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_4bits_merged_model_load_8bits_results") +# ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) + +# Model comparison report +#print model comparison +ocr_evaluator.print_model_comparison() + + + +# Final cleanup +print("\n🧹 Cleaning up temporary files...") +safe_remove_directory("./unsloth-qwen2.5-vl-32b-french-ocr-adapter") +safe_remove_directory("./unsloth-qwen2.5-vl-32b-french-ocr-checkpoints") +safe_remove_directory("./unsloth_compiled_cache") +safe_remove_directory("./qwen2.5-ocr-merged-finetune-merge-16bit") + +print("\n🎯 Pipeline completed successfully!") +print("=" * 80) diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index f559c6c01e..28fa163e65 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -618,6 +618,11 @@ __INT_TO_FLOAT_MAPPER = \ "Qwen/Qwen2.5-VL-7B-Instruct", "unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit", ), + "unsloth/Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit" : ( + "unsloth/Qwen2.5-VL-32B-Instruct", + "Qwen/Qwen2.5-VL-32B-Instruct", + "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit", + ), "unsloth/Qwen2.5-VL-72B-Instruct-unsloth-bnb-4bit" : ( "unsloth/Qwen2.5-VL-72B-Instruct", "Qwen/Qwen2.5-VL-72B-Instruct", From 663100749356af2539b03fc5c8984ebf183893ac Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 17 Jul 2025 15:35:28 -0700 Subject: [PATCH 5/5] =?UTF-8?q?Revert=20"Add=20Qwen2.5-VL-32B-Instruct=20m?= =?UTF-8?q?apping=20to=20fix=20quantized=20model=20merge=20erro=E2=80=A6"?= =?UTF-8?q?=20(#2988)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 73c02231dee188180d6678d24a4017322eea416f. --- ..._merge_qwen2.5vl32B_model_ocr_benchmark.py | 255 ------------------ unsloth/models/mapper.py | 5 - 2 files changed, 260 deletions(-) delete mode 100644 tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py diff --git a/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py b/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py deleted file mode 100644 index 0bf548b41c..0000000000 --- a/tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py +++ /dev/null @@ -1,255 +0,0 @@ -# -*- coding: utf-8 -*- - -from unsloth import FastVisionModel - -import torch -from qwen_vl_utils import process_vision_info -import os -from datasets import load_dataset -from trl import SFTTrainer, SFTConfig - -import sys -from pathlib import Path - - -REPO_ROOT = Path(__file__).parents[3] -sys.path.insert(0, str(REPO_ROOT)) - -from tests.utils.cleanup_utils import safe_remove_directory -from tests.utils.ocr_eval import OCRModelEvaluator - - -## Dataset Preparation -from datasets import load_dataset - -dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", 'en', split="train") -# To select the first 2000 examples -train_dataset = dataset.select(range(2000)) - -# To select the next 200 examples for evaluation -eval_dataset = dataset.select(range(2000, 2200)) - -# Convert dataset to OAI messages -def format_data(sample): - return {"messages": [ - { - "role": "system", - "content": [{"type": "text", "text": system_message}], - }, - { - "role": "user", - "content": [ - { - "type": "text", - "text": sample["question"], - },{ - "type": "image", - "image": sample["image"], - } - ], - }, - { - "role": "assistant", - "content": [{"type": "text", "text": sample["answer"]}], - }, - ], - } - -system_message = "You are an expert french ocr system." -# Convert dataset to OAI messages -# need to use list comprehension to keep Pil.Image type, .mape convert image to bytes -train_dataset = [format_data(sample) for sample in train_dataset] -eval_dataset = [format_data(sample) for sample in eval_dataset] - -## Setup OCR main evaluation function and helpers -import os -import torch -from tqdm import tqdm -import pandas as pd -from jiwer import wer, cer -from qwen_vl_utils import process_vision_info - -# -ocr_evaluator = OCRModelEvaluator() -model_comparison_results = {} - -## Finetuning Setup and Run -# Load Base Model - -model, tokenizer = FastVisionModel.from_pretrained( - model_name = "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit", - max_seq_length = 2048, # Choose any for long context! - load_in_4bit = True, # 4 bit quantization to reduce memory - load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory - full_finetuning = False, # [NEW!] We have full finetuning now! -) - -# benchmark base model performance -model_name = "Unsloth Base model" -FastVisionModel.for_inference(model) -avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_base_model_results") -ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) - -## Lora Finetuning -model = FastVisionModel.get_peft_model( - model, - finetune_vision_layers = True, # Turn off for just text! - finetune_language_layers = True, # Should leave on! - finetune_attention_modules = True, # Attention good for GRPO - finetune_mlp_modules = True, # SHould leave on always! - - r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128 - #target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", - #"gate_proj", "up_proj", "down_proj",], - lora_alpha = 32, - lora_dropout = 0, # Supports any, but = 0 is optimized - bias = "none", # Supports any, but = "none" is optimized - # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes! - use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context - random_state = 3407, - use_rslora = False, # We support rank stabilized LoRA - loftq_config = None, # And LoftQ -) - -from unsloth import is_bf16_supported -from unsloth.trainer import UnslothVisionDataCollator -FastVisionModel.for_training(model) # Enable for training! -model.config.use_cache = False - - -trainer = SFTTrainer( - model = model, - tokenizer = tokenizer, - data_collator = UnslothVisionDataCollator(model, tokenizer), - train_dataset = train_dataset, - args = SFTConfig( - #per_device_train_batch_size = 4, - #gradient_accumulation_steps = 8, - per_device_train_batch_size = 2, - gradient_accumulation_steps = 4, - gradient_checkpointing=True, - gradient_checkpointing_kwargs = {"use_reentrant": False}, # use reentrant checkpointing - max_grad_norm=0.3, # max gradient norm based on QLoRA paper - warmup_ratio=0.03, - #num_train_epochs = 2, # Set this instead of max_steps for full training runs - max_steps=60, - learning_rate = 2e-4, - fp16 = not is_bf16_supported(), - bf16 = is_bf16_supported(), - logging_steps = 5, - save_strategy="epoch", - optim = "adamw_torch_fused", - weight_decay = 0.01, - lr_scheduler_type = "linear", - seed = 3407, - output_dir = "unsloth-qwen2.5-vl-32b-french-ocr-checkpoints", - report_to = "none", # For Weights and Biases - - # You MUST put the below items for vision finetuning: - remove_unused_columns = False, - dataset_text_field = "", - dataset_kwargs = {"skip_prepare_dataset": True}, - dataset_num_proc = 4, - max_seq_length = 2048, - ), -) - -# run training -trainer_stats = trainer.train() - -model.save_pretrained("unsloth-qwen2.5-vl-32b-french-ocr-adapter", tokenizer) -tokenizer.save_pretrained("unsloth-qwen2.5-vl-32b-french-ocr-adapter") - -## Measure Adapter Performance - -# benchmark lora model performance -model_name = "Unsloth lora adapter model" -FastVisionModel.for_inference(model) -avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_lora_model_results") -ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) - -## Merge Model - -def find_lora_base_model(model_to_inspect): - current = model_to_inspect - if hasattr(current, "base_model"): - current = current.base_model - if hasattr(current, "model"): - current = current.model - return current -pass - -base = find_lora_base_model(model) - -print((base.__class__.__name__)) - -# merge default 16 bits -model.save_pretrained_merged(save_directory="qwen2.5-ocr-merged-finetune-merge-16bit", tokenizer=tokenizer) - - -## Benchmark merged model performance - -### 16 bits merged model - -model, tokenizer = FastVisionModel.from_pretrained("./qwen2.5-ocr-merged-finetune-merge-16bit",load_in_4bit=False, load_in_8bit=False) - -# benchmark 4bit loaded, 16bits merged model performance -model_name = "Unsloth 16bits-merged model load-16bits" -model.config.use_cache = True - -avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_16bits_merged_model_load_16bits_results") -ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) - -# load 16bits-merged model in 4 bits -model, tokenizer = FastVisionModel.from_pretrained("./qwen2.5-ocr-merged-finetune-merge-16bit",load_in_4bit=True, load_in_8bit=False) - -# benchmark 4bit loaded, 16bits merged model performance -model_name = "Unsloth 16bits-merged model load-4bits" -model.config.use_cache = True - -avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_16bits_merged_model_load_4bits_results") -ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) - -# load model in 8 bits -model, tokenizer = FastVisionModel.from_pretrained("./qwen2.5-ocr-merged-finetune-merge-16bit",load_in_4bit=False, load_in_8bit=True) - -# benchmark 4bit loaded, 16bits merged model performance -model_name = "Unsloth 16bits-merged model load-8bits" -avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_16bits_merged_model_load_8bits_results") -ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) - -# """### 4 bits merged model""" -# -# # load 4bits-merged model in 4 bits -# model, tokenizer = FastVisionModel.from_pretrained("./qwen2-ocr-merged-finetune-merge-4bit",load_in_4bit=True, load_in_8bit=False) -# -# # benchmark 4bit loaded, 4bits merged model performance -# model_name = "Unsloth 4bits-merged model load-4bits" -# -# avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_4bits_merged_model_load_4bits_results") -# ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) -# -# # load model in 8 bits -# model, tokenizer = FastVisionModel.from_pretrained("./qwen2-ocr-merged-finetune-merge-4bit",load_in_4bit=False, load_in_8bit=True) -# -# # benchmark 8bit loaded, 4bits merged model performance -# model_name = "Unsloth 4bits-merged model load-8bits" -# -# avg_wer, avg_cer = ocr_evaluator.evaluate_model(model, tokenizer, eval_dataset, output_dir="unsloth_4bits_merged_model_load_8bits_results") -# ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer) - -# Model comparison report -#print model comparison -ocr_evaluator.print_model_comparison() - - - -# Final cleanup -print("\n🧹 Cleaning up temporary files...") -safe_remove_directory("./unsloth-qwen2.5-vl-32b-french-ocr-adapter") -safe_remove_directory("./unsloth-qwen2.5-vl-32b-french-ocr-checkpoints") -safe_remove_directory("./unsloth_compiled_cache") -safe_remove_directory("./qwen2.5-ocr-merged-finetune-merge-16bit") - -print("\n🎯 Pipeline completed successfully!") -print("=" * 80) diff --git a/unsloth/models/mapper.py b/unsloth/models/mapper.py index 28fa163e65..f559c6c01e 100644 --- a/unsloth/models/mapper.py +++ b/unsloth/models/mapper.py @@ -618,11 +618,6 @@ __INT_TO_FLOAT_MAPPER = \ "Qwen/Qwen2.5-VL-7B-Instruct", "unsloth/Qwen2.5-VL-7B-Instruct-bnb-4bit", ), - "unsloth/Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit" : ( - "unsloth/Qwen2.5-VL-32B-Instruct", - "Qwen/Qwen2.5-VL-32B-Instruct", - "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit", - ), "unsloth/Qwen2.5-VL-72B-Instruct-unsloth-bnb-4bit" : ( "unsloth/Qwen2.5-VL-72B-Instruct", "Qwen/Qwen2.5-VL-72B-Instruct",