ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[pre-commit.ci] auto fixes from pre-commit.com hooks

Browse source 
for more information, see https://pre-commit.ci
This commit is contained in:
pre-commit-ci[bot] 2025-12-01 13:33:35 +00:00
commit fbb98c5c5c
47 changed files with 2646 additions and 2646 deletions
Download patch file Download diff file Expand all files Collapse all files

18
tests/saving/gpt-oss-merge/test_merged_model.py
View file

@ -22,10 +22,10 @@ def safe_remove_directory(path):
print("🔥 Loading the 16-bit merged model from disk...")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./gpt-oss-finetuned-merged",
max_seq_length = 1024,
load_in_4bit = True,
load_in_8bit = False,
model_name="./gpt-oss-finetuned-merged",
max_seq_length=1024,
load_in_4bit=True,
load_in_8bit=False,
)
print("✅ Merged model loaded successfully.")
@ -36,14 +36,14 @@ messages = [
]
inputs = merged_tokenizer.apply_chat_template(
messages,
add_generation_prompt = True,
return_tensors = "pt",
return_dict = True,
reasoning_effort = "low", # **NEW!** Set reasoning effort to low, medium or high
add_generation_prompt=True,
return_tensors="pt",
return_dict=True,
reasoning_effort="low", # **NEW!** Set reasoning effort to low, medium or high
).to(merged_model.device)
_ = merged_model.generate(
**inputs, max_new_tokens = 512, streamer = TextStreamer(merged_tokenizer)
**inputs, max_new_tokens=512, streamer=TextStreamer(merged_tokenizer)
)
print("\n✅ Inference complete.")

40
tests/saving/gpt-oss-merge/train_and_merge.py
View file

@ -29,7 +29,7 @@ def formatting_prompts_func(examples):
convos = examples["messages"]
texts = [
tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
@ -40,17 +40,17 @@ def formatting_prompts_func(examples):
print("Loading 4-bit Mxfp4 gpt-oss model for training...")
max_seq_length = 1024
model, tokenizer = FastLanguageModel.from_pretrained(
"unsloth/gpt-oss-20b", max_seq_length = max_seq_length, load_in_4bit = True
"unsloth/gpt-oss-20b", max_seq_length=max_seq_length, load_in_4bit=True
)
dataset = load_dataset("HuggingFaceH4/Multilingual-Thinking", split = "train[:50]").map(
formatting_prompts_func, batched = True
dataset = load_dataset("HuggingFaceH4/Multilingual-Thinking", split="train[:50]").map(
formatting_prompts_func, batched=True
)
model = FastLanguageModel.get_peft_model(
model,
r = 8,
target_modules = [
r=8,
target_modules=[
"q_proj",
"k_proj",
"v_proj",
@ -59,22 +59,22 @@ model = FastLanguageModel.get_peft_model(
"up_proj",
"down_proj",
],
lora_alpha = 16,
use_gradient_checkpointing = "unsloth",
random_state = 3407,
lora_alpha=16,
use_gradient_checkpointing="unsloth",
random_state=3407,
)
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset,
args = SFTConfig(
per_device_train_batch_size = 1,
gradient_accumulation_steps = 4,
max_steps = 10,
learning_rate = 2e-4,
output_dir = "outputs",
report_to = "none",
model=model,
tokenizer=tokenizer,
train_dataset=dataset,
args=SFTConfig(
per_device_train_batch_size=1,
gradient_accumulation_steps=4,
max_steps=10,
learning_rate=2e-4,
output_dir="outputs",
report_to="none",
),
)
@ -85,7 +85,7 @@ print("Fine-tuning complete.")
# --- Merge and Save ---
print("\n💾 Merging and saving the 16-bit model to './gpt-oss-finetuned-merged'...")
model.save_pretrained_merged(
save_directory = "./gpt-oss-finetuned-merged", tokenizer = tokenizer
save_directory="./gpt-oss-finetuned-merged", tokenizer=tokenizer
)
print("✅ Model merged and saved.")

172
tests/saving/language_models/test_merge_4bit_validation.py
View file

@ -17,7 +17,7 @@ def formatting_prompts_func(examples):
convos = examples["messages"]
texts = [
tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
@ -36,25 +36,25 @@ else:
attn_implementation = "sdpa"
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/Llama-3.1-8B-Instruct",
max_seq_length = 2048,
dtype = compute_dtype,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
attn_implementation = attn_implementation,
model_name="unsloth/Llama-3.1-8B-Instruct",
max_seq_length=2048,
dtype=compute_dtype,
load_in_4bit=True,
load_in_8bit=False,
full_finetuning=False,
attn_implementation=attn_implementation,
)
tokenizer = get_chat_template(
tokenizer,
chat_template = "llama-3.1",
chat_template="llama-3.1",
)
# Load small dataset for quick training
dataset_train = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "train[:100]"
"allenai/openassistant-guanaco-reformatted", split="train[:100]"
)
dataset_train = dataset_train.map(formatting_prompts_func, batched = True)
dataset_train = dataset_train.map(formatting_prompts_func, batched=True)
print("✅ Base model loaded successfully!")
@ -64,8 +64,8 @@ print(f"{'='*80}")
model = FastLanguageModel.get_peft_model(
model,
r = 16,
target_modules = [
r=16,
target_modules=[
"k_proj",
"q_proj",
"v_proj",
@ -74,40 +74,40 @@ model = FastLanguageModel.get_peft_model(
"down_proj",
"up_proj",
],
lora_alpha = 16,
lora_dropout = 0,
bias = "none",
use_gradient_checkpointing = "unsloth",
random_state = 3407,
use_rslora = False,
loftq_config = None,
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=3407,
use_rslora=False,
loftq_config=None,
)
from unsloth import is_bfloat16_supported
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset_train,
dataset_text_field = "text",
max_seq_length = 2048,
data_collator = DataCollatorForSeq2Seq(tokenizer = tokenizer),
dataset_num_proc = 2,
packing = False,
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
warmup_ratio = 0.1,
max_steps = 10, # Very short training for test
learning_rate = 2e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 5,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "outputs",
report_to = "none",
model=model,
tokenizer=tokenizer,
train_dataset=dataset_train,
dataset_text_field="text",
max_seq_length=2048,
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer),
dataset_num_proc=2,
packing=False,
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
warmup_ratio=0.1,
max_steps=10, # Very short training for test
learning_rate=2e-4,
fp16=not is_bfloat16_supported(),
bf16=is_bfloat16_supported(),
logging_steps=5,
optim="adamw_8bit",
lr_scheduler_type="linear",
seed=3407,
output_dir="outputs",
report_to="none",
),
)
@ -119,9 +119,9 @@ print("🔍 PHASE 3: Save with Forced 4bit Merge")
print(f"{'='*80}")
model.save_pretrained_merged(
save_directory = "./test_4bit_model",
tokenizer = tokenizer,
save_method = "forced_merged_4bit",
save_directory="./test_4bit_model",
tokenizer=tokenizer,
save_method="forced_merged_4bit",
)
print("✅ Model saved with forced 4bit merge!")
@ -137,15 +137,15 @@ torch.cuda.empty_cache()
# Load the 4bit merged model
model_4bit, tokenizer_4bit = FastLanguageModel.from_pretrained(
model_name = "./test_4bit_model",
max_seq_length = 2048,
load_in_4bit = True,
load_in_8bit = False,
model_name="./test_4bit_model",
max_seq_length=2048,
load_in_4bit=True,
load_in_8bit=False,
)
tokenizer_4bit = get_chat_template(
tokenizer_4bit,
chat_template = "llama-3.1",
chat_template="llama-3.1",
)
print("✅ 4bit model loaded successfully!")
@ -153,8 +153,8 @@ print("✅ 4bit model loaded successfully!")
# Add LoRA adapters to the 4bit model
model_4bit = FastLanguageModel.get_peft_model(
model_4bit,
r = 16,
target_modules = [
r=16,
target_modules=[
"k_proj",
"q_proj",
"v_proj",
@ -163,39 +163,39 @@ model_4bit = FastLanguageModel.get_peft_model(
"down_proj",
"up_proj",
],
lora_alpha = 16,
lora_dropout = 0,
bias = "none",
use_gradient_checkpointing = "unsloth",
random_state = 3407,
use_rslora = False,
loftq_config = None,
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=3407,
use_rslora=False,
loftq_config=None,
)
# Second fine-tuning
trainer_4bit = SFTTrainer(
model = model_4bit,
tokenizer = tokenizer_4bit,
train_dataset = dataset_train,
dataset_text_field = "text",
max_seq_length = 2048,
data_collator = DataCollatorForSeq2Seq(tokenizer = tokenizer_4bit),
dataset_num_proc = 2,
packing = False,
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
warmup_ratio = 0.1,
max_steps = 10, # Very short training for test
learning_rate = 2e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 5,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "outputs_4bit",
report_to = "none",
model=model_4bit,
tokenizer=tokenizer_4bit,
train_dataset=dataset_train,
dataset_text_field="text",
max_seq_length=2048,
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer_4bit),
dataset_num_proc=2,
packing=False,
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
warmup_ratio=0.1,
max_steps=10, # Very short training for test
learning_rate=2e-4,
fp16=not is_bfloat16_supported(),
bf16=is_bfloat16_supported(),
logging_steps=5,
optim="adamw_8bit",
lr_scheduler_type="linear",
seed=3407,
output_dir="outputs_4bit",
report_to="none",
),
)
@ -208,8 +208,8 @@ print(f"{'='*80}")
try:
model_4bit.save_pretrained_merged(
save_directory = "./test_should_fail",
tokenizer = tokenizer_4bit,
save_directory="./test_should_fail",
tokenizer=tokenizer_4bit,
# No save_method specified, should default to regular merge
)
assert False, "Expected TypeError but merge succeeded!"
@ -225,9 +225,9 @@ print(f"{'='*80}")
try:
model_4bit.save_pretrained_merged(
save_directory = "./test_4bit_second",
tokenizer = tokenizer_4bit,
save_method = "forced_merged_4bit",
save_directory="./test_4bit_second",
tokenizer=tokenizer_4bit,
save_method="forced_merged_4bit",
)
print("✅ Successfully saved 4bit model with forced 4bit method!")
except Exception as e:

132
tests/saving/language_models/test_merge_model_perplexity_llama-3.2.py
View file

@ -36,14 +36,14 @@ def formatting_prompts_func(examples):
convos = examples["messages"]
texts = [
tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
return {"text": texts}
def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit = False):
def load_and_compute_8bit_ppl(result_queue, load_in_4bit=False, load_in_8bit=False):
"""Load model and compute perplexity in subprocess"""
from unsloth import FastLanguageModel
from unsloth.chat_templates import get_chat_template
@ -51,20 +51,20 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Load model
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_llama_text_model",
max_seq_length = 2048,
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
model_name="./unsloth_out/merged_llama_text_model",
max_seq_length=2048,
load_in_4bit=load_in_4bit,
load_in_8bit=load_in_8bit,
)
# Set up tokenizer
merged_tokenizer = get_chat_template(
merged_tokenizer,
chat_template = "llama-3.1",
chat_template="llama-3.1",
)
# Load dataset fresh in subprocess
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
# Format the dataset
@ -72,13 +72,13 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
convos = examples["messages"]
texts = [
merged_tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
return {"text": texts}
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
# Compute perplexity using the passed dataset
ppl_value = ppl_model(merged_model, merged_tokenizer, dataset_ppl)
@ -104,7 +104,7 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Main execution code should be wrapped in this guard
if __name__ == "__main__":
mp.set_start_method("spawn", force = True)
mp.set_start_method("spawn", force=True)
if torch.cuda.is_bf16_supported():
compute_dtype = torch.bfloat16
@ -114,38 +114,38 @@ if __name__ == "__main__":
attn_implementation = "sdpa"
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/Llama-3.2-3B-Instruct",
max_seq_length = 2048,
dtype = compute_dtype,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
attn_implementation = attn_implementation,
model_name="unsloth/Llama-3.2-3B-Instruct",
max_seq_length=2048,
dtype=compute_dtype,
load_in_4bit=True,
load_in_8bit=False,
full_finetuning=False,
attn_implementation=attn_implementation,
)
tokenizer = get_chat_template(
tokenizer,
chat_template = "llama-3.1",
chat_template="llama-3.1",
)
from unsloth.chat_templates import standardize_sharegpt
dataset_train = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "train"
"allenai/openassistant-guanaco-reformatted", split="train"
)
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
dataset_train = dataset_train.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_train = dataset_train.map(formatting_prompts_func, batched=True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
add_to_comparison("Base model 4 bits", ppl_model(model, tokenizer, dataset_ppl))
model = FastLanguageModel.get_peft_model(
model,
r = 16,
target_modules = [
r=16,
target_modules=[
"k_proj",
"q_proj",
"v_proj",
@ -154,40 +154,40 @@ if __name__ == "__main__":
"down_proj",
"up_proj",
],
lora_alpha = 16,
lora_dropout = 0,
bias = "none",
use_gradient_checkpointing = "unsloth",
random_state = 3407,
use_rslora = False,
loftq_config = None,
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=3407,
use_rslora=False,
loftq_config=None,
)
from unsloth import is_bfloat16_supported
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset_train,
dataset_text_field = "text",
max_seq_length = 2048,
data_collator = DataCollatorForSeq2Seq(tokenizer = tokenizer),
dataset_num_proc = 2,
packing = False,
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
warmup_ratio = 0.1,
max_steps = 10,
learning_rate = 2e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 50,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "outputs",
report_to = "none",
model=model,
tokenizer=tokenizer,
train_dataset=dataset_train,
dataset_text_field="text",
max_seq_length=2048,
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer),
dataset_num_proc=2,
packing=False,
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
warmup_ratio=0.1,
max_steps=10,
learning_rate=2e-4,
fp16=not is_bfloat16_supported(),
bf16=is_bfloat16_supported(),
logging_steps=50,
optim="adamw_8bit",
lr_scheduler_type="linear",
seed=3407,
output_dir="outputs",
report_to="none",
),
)
@ -195,8 +195,8 @@ if __name__ == "__main__":
trainer = train_on_responses_only(
trainer,
instruction_part = "<|start_header_id|>user<|end_header_id|>\n\n",
response_part = "<|start_header_id|>assistant<|end_header_id|>\n\n",
instruction_part="<|start_header_id|>user<|end_header_id|>\n\n",
response_part="<|start_header_id|>assistant<|end_header_id|>\n\n",
)
# run training
@ -207,7 +207,7 @@ if __name__ == "__main__":
# saving and merging the model to local disk
print("merge and save to local disk")
model.save_pretrained_merged(
save_directory = "./unsloth_out/merged_llama_text_model", tokenizer = tokenizer
save_directory="./unsloth_out/merged_llama_text_model", tokenizer=tokenizer
)
# print("cleaning")
@ -219,10 +219,10 @@ if __name__ == "__main__":
# load model from local disk and test
print("Loading merged model in 4 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_llama_text_model",
max_seq_length = 2048,
load_in_4bit = True,
load_in_8bit = False,
model_name="./unsloth_out/merged_llama_text_model",
max_seq_length=2048,
load_in_4bit=True,
load_in_8bit=False,
)
add_to_comparison(
@ -231,7 +231,7 @@ if __name__ == "__main__":
print("Computing 8-bit model perplexity in subprocess...")
result_queue = mp.Queue()
p = mp.Process(target = load_and_compute_8bit_ppl, args = (result_queue, False, True))
p = mp.Process(target=load_and_compute_8bit_ppl, args=(result_queue, False, True))
p.start()
p.join()
@ -240,10 +240,10 @@ if __name__ == "__main__":
print("Loading merged model in 16 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_llama_text_model",
max_seq_length = 2048,
load_in_4bit = False,
load_in_8bit = False,
model_name="./unsloth_out/merged_llama_text_model",
max_seq_length=2048,
load_in_4bit=False,
load_in_8bit=False,
)
add_to_comparison(

118
tests/saving/language_models/test_merge_model_perplexity_mistral.py
View file

@ -30,17 +30,17 @@ from tests.utils.perplexity_eval import (
)
def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit = False):
def load_and_compute_8bit_ppl(result_queue, load_in_4bit=False, load_in_8bit=False):
"""Load model and compute perplexity in subprocess"""
from unsloth import FastLanguageModel
from tests.utils.perplexity_eval import ppl_model
# Load model
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_mistral_text_model",
max_seq_length = 2048,
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
model_name="./unsloth_out/merged_mistral_text_model",
max_seq_length=2048,
load_in_4bit=load_in_4bit,
load_in_8bit=load_in_8bit,
)
# Set up tokenizer
# merged_tokenizer = get_chat_template(
@ -50,7 +50,7 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Load dataset fresh in subprocess
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
@ -103,7 +103,7 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
"text": texts,
}
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
# Compute perplexity using the passed dataset
ppl_value = ppl_model(merged_model, merged_tokenizer, dataset_ppl)
@ -129,7 +129,7 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Main execution code should be wrapped in this guard
if __name__ == "__main__":
mp.set_start_method("spawn", force = True)
mp.set_start_method("spawn", force=True)
if torch.cuda.is_bf16_supported():
compute_dtype = torch.bfloat16
@ -139,13 +139,13 @@ if __name__ == "__main__":
attn_implementation = "sdpa"
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/mistral-7b-v0.3",
max_seq_length = 2048,
dtype = compute_dtype,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
attn_implementation = attn_implementation,
model_name="unsloth/mistral-7b-v0.3",
max_seq_length=2048,
dtype=compute_dtype,
load_in_4bit=True,
load_in_8bit=False,
full_finetuning=False,
attn_implementation=attn_implementation,
)
EOS_TOKEN = tokenizer.eos_token
@ -200,21 +200,21 @@ if __name__ == "__main__":
}
dataset_train = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "train"
"allenai/openassistant-guanaco-reformatted", split="train"
)
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
dataset_train = dataset_train.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_train = dataset_train.map(formatting_prompts_func, batched=True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
add_to_comparison("Base model 4 bits", ppl_model(model, tokenizer, dataset_ppl))
model = FastLanguageModel.get_peft_model(
model,
r = 16,
target_modules = [
r=16,
target_modules=[
"k_proj",
"q_proj",
"v_proj",
@ -223,39 +223,39 @@ if __name__ == "__main__":
"down_proj",
"up_proj",
],
lora_alpha = 16,
lora_dropout = 0,
bias = "none",
use_gradient_checkpointing = "unsloth",
random_state = 3407,
use_rslora = False,
loftq_config = None,
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=3407,
use_rslora=False,
loftq_config=None,
)
from unsloth import is_bfloat16_supported
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset_train,
dataset_text_field = "text",
max_seq_length = 2048,
dataset_num_proc = 2,
packing = False,
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
warmup_ratio = 0.1,
max_steps = 200,
learning_rate = 2e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 50,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "outputs",
report_to = "none",
model=model,
tokenizer=tokenizer,
train_dataset=dataset_train,
dataset_text_field="text",
max_seq_length=2048,
dataset_num_proc=2,
packing=False,
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
warmup_ratio=0.1,
max_steps=200,
learning_rate=2e-4,
fp16=not is_bfloat16_supported(),
bf16=is_bfloat16_supported(),
logging_steps=50,
optim="adamw_8bit",
lr_scheduler_type="linear",
seed=3407,
output_dir="outputs",
report_to="none",
),
)
@ -267,7 +267,7 @@ if __name__ == "__main__":
# saving and merging the model to local disk
print("merge and save to local disk")
model.save_pretrained_merged(
save_directory = "./unsloth_out/merged_mistral_text_model", tokenizer = tokenizer
save_directory="./unsloth_out/merged_mistral_text_model", tokenizer=tokenizer
)
# print("cleaning")
@ -279,10 +279,10 @@ if __name__ == "__main__":
# load model from local disk and test
print("Loading merged model in 4 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_mistral_text_model",
max_seq_length = 2048,
load_in_4bit = True,
load_in_8bit = False,
model_name="./unsloth_out/merged_mistral_text_model",
max_seq_length=2048,
load_in_4bit=True,
load_in_8bit=False,
)
add_to_comparison(
@ -291,7 +291,7 @@ if __name__ == "__main__":
print("Computing 8-bit model perplexity in subprocess...")
result_queue = mp.Queue()
p = mp.Process(target = load_and_compute_8bit_ppl, args = (result_queue, False, True))
p = mp.Process(target=load_and_compute_8bit_ppl, args=(result_queue, False, True))
p.start()
p.join()
@ -300,10 +300,10 @@ if __name__ == "__main__":
print("Loading merged model in 16 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_mistral_text_model",
max_seq_length = 2048,
load_in_4bit = False,
load_in_8bit = False,
model_name="./unsloth_out/merged_mistral_text_model",
max_seq_length=2048,
load_in_4bit=False,
load_in_8bit=False,
)
add_to_comparison(

132
tests/saving/language_models/test_merge_model_perplexity_phi_4.py
View file

@ -36,7 +36,7 @@ def formatting_prompts_func(examples):
convos = examples["messages"]
texts = [
tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
@ -45,7 +45,7 @@ def formatting_prompts_func(examples):
}
def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit = False):
def load_and_compute_8bit_ppl(result_queue, load_in_4bit=False, load_in_8bit=False):
"""Load model and compute perplexity in subprocess"""
from unsloth import FastLanguageModel
from unsloth.chat_templates import get_chat_template
@ -53,20 +53,20 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Load model
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_phi4_text_model",
max_seq_length = 2048,
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
model_name="./unsloth_out/merged_phi4_text_model",
max_seq_length=2048,
load_in_4bit=load_in_4bit,
load_in_8bit=load_in_8bit,
)
# Set up tokenizer
merged_tokenizer = get_chat_template(
merged_tokenizer,
chat_template = "phi-4",
chat_template="phi-4",
)
# Load dataset fresh in subprocess
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
# Format the dataset
@ -74,13 +74,13 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
convos = examples["messages"]
texts = [
merged_tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
return {"text": texts}
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
# Compute perplexity using the passed dataset
ppl_value = ppl_model(merged_model, merged_tokenizer, dataset_ppl)
@ -106,7 +106,7 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Main execution code should be wrapped in this guard
if __name__ == "__main__":
mp.set_start_method("spawn", force = True)
mp.set_start_method("spawn", force=True)
if torch.cuda.is_bf16_supported():
compute_dtype = torch.bfloat16
@ -116,36 +116,36 @@ if __name__ == "__main__":
attn_implementation = "sdpa"
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/Phi-4",
max_seq_length = 2048,
dtype = compute_dtype,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
attn_implementation = attn_implementation,
model_name="unsloth/Phi-4",
max_seq_length=2048,
dtype=compute_dtype,
load_in_4bit=True,
load_in_8bit=False,
full_finetuning=False,
attn_implementation=attn_implementation,
)
tokenizer = get_chat_template(
tokenizer,
chat_template = "phi-4",
chat_template="phi-4",
)
dataset_train = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "train"
"allenai/openassistant-guanaco-reformatted", split="train"
)
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
dataset_train = dataset_train.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_train = dataset_train.map(formatting_prompts_func, batched=True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
add_to_comparison("Base model 4 bits", ppl_model(model, tokenizer, dataset_ppl))
model = FastLanguageModel.get_peft_model(
model,
r = 16,
target_modules = [
r=16,
target_modules=[
"k_proj",
"q_proj",
"v_proj",
@ -154,40 +154,40 @@ if __name__ == "__main__":
"down_proj",
"up_proj",
],
lora_alpha = 16,
lora_dropout = 0,
bias = "none",
use_gradient_checkpointing = "unsloth",
random_state = 3407,
use_rslora = False,
loftq_config = None,
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=3407,
use_rslora=False,
loftq_config=None,
)
from unsloth import is_bfloat16_supported
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset_train,
dataset_text_field = "text",
max_seq_length = 2048,
data_collator = DataCollatorForSeq2Seq(tokenizer = tokenizer),
dataset_num_proc = 2,
packing = False,
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
warmup_ratio = 0.1,
max_steps = 200,
learning_rate = 2e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 50,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "outputs",
report_to = "none",
model=model,
tokenizer=tokenizer,
train_dataset=dataset_train,
dataset_text_field="text",
max_seq_length=2048,
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer),
dataset_num_proc=2,
packing=False,
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
warmup_ratio=0.1,
max_steps=200,
learning_rate=2e-4,
fp16=not is_bfloat16_supported(),
bf16=is_bfloat16_supported(),
logging_steps=50,
optim="adamw_8bit",
lr_scheduler_type="linear",
seed=3407,
output_dir="outputs",
report_to="none",
),
)
@ -195,8 +195,8 @@ if __name__ == "__main__":
trainer = train_on_responses_only(
trainer,
instruction_part = "<|im_start|>user<|im_sep|>\n\n",
response_part = "<|im_start|>assistant<|im_sep|>\n\n",
instruction_part="<|im_start|>user<|im_sep|>\n\n",
response_part="<|im_start|>assistant<|im_sep|>\n\n",
)
# run training
@ -207,7 +207,7 @@ if __name__ == "__main__":
# saving and merging the model to local disk
print("merge and save to local disk")
model.save_pretrained_merged(
save_directory = "./unsloth_out/merged_phi4_text_model", tokenizer = tokenizer
save_directory="./unsloth_out/merged_phi4_text_model", tokenizer=tokenizer
)
# print("cleaning")
@ -219,10 +219,10 @@ if __name__ == "__main__":
# load model from local disk and test
print("Loading merged model in 4 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_phi4_text_model",
max_seq_length = 2048,
load_in_4bit = True,
load_in_8bit = False,
model_name="./unsloth_out/merged_phi4_text_model",
max_seq_length=2048,
load_in_4bit=True,
load_in_8bit=False,
)
add_to_comparison(
@ -231,7 +231,7 @@ if __name__ == "__main__":
print("Computing 8-bit model perplexity in subprocess...")
result_queue = mp.Queue()
p = mp.Process(target = load_and_compute_8bit_ppl, args = (result_queue, False, True))
p = mp.Process(target=load_and_compute_8bit_ppl, args=(result_queue, False, True))
p.start()
p.join()
@ -240,10 +240,10 @@ if __name__ == "__main__":
print("Loading merged model in 16 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_phi4_text_model",
max_seq_length = 2048,
load_in_4bit = False,
load_in_8bit = False,
model_name="./unsloth_out/merged_phi4_text_model",
max_seq_length=2048,
load_in_4bit=False,
load_in_8bit=False,
)
add_to_comparison(

132
tests/saving/language_models/test_merged_model_perplexity_llama-3.1-8b.py
View file

@ -35,14 +35,14 @@ def formatting_prompts_func(examples):
convos = examples["messages"]
texts = [
tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
return {"text": texts}
def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit = False):
def load_and_compute_8bit_ppl(result_queue, load_in_4bit=False, load_in_8bit=False):
"""Load model and compute perplexity in subprocess"""
from unsloth import FastLanguageModel
from unsloth.chat_templates import get_chat_template
@ -50,20 +50,20 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Load model
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_llama_text_model",
max_seq_length = 2048,
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
model_name="./unsloth_out/merged_llama_text_model",
max_seq_length=2048,
load_in_4bit=load_in_4bit,
load_in_8bit=load_in_8bit,
)
# Set up tokenizer
merged_tokenizer = get_chat_template(
merged_tokenizer,
chat_template = "llama-3.1",
chat_template="llama-3.1",
)
# Load dataset fresh in subprocess
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
# Format the dataset
@ -71,13 +71,13 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
convos = examples["messages"]
texts = [
merged_tokenizer.apply_chat_template(
convo, tokenize = False, add_generation_prompt = False
convo, tokenize=False, add_generation_prompt=False
)
for convo in convos
]
return {"text": texts}
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
# Compute perplexity using the passed dataset
ppl_value = ppl_model(merged_model, merged_tokenizer, dataset_ppl)
@ -103,7 +103,7 @@ def load_and_compute_8bit_ppl(result_queue, load_in_4bit = False, load_in_8bit =
# Main execution code should be wrapped in this guard
if __name__ == "__main__":
mp.set_start_method("spawn", force = True)
mp.set_start_method("spawn", force=True)
if torch.cuda.is_bf16_supported():
compute_dtype = torch.bfloat16
@ -113,31 +113,31 @@ if __name__ == "__main__":
attn_implementation = "sdpa"
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/Llama-3.1-8B-Instruct",
max_seq_length = 2048,
dtype = compute_dtype,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
attn_implementation = attn_implementation,
model_name="unsloth/Llama-3.1-8B-Instruct",
max_seq_length=2048,
dtype=compute_dtype,
load_in_4bit=True,
load_in_8bit=False,
full_finetuning=False,
attn_implementation=attn_implementation,
)
tokenizer = get_chat_template(
tokenizer,
chat_template = "llama-3.1",
chat_template="llama-3.1",
)
from unsloth.chat_templates import standardize_sharegpt
dataset_train = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "train"
"allenai/openassistant-guanaco-reformatted", split="train"
)
dataset_ppl = load_dataset(
"allenai/openassistant-guanaco-reformatted", split = "eval"
"allenai/openassistant-guanaco-reformatted", split="eval"
)
dataset_train = dataset_train.map(formatting_prompts_func, batched = True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched = True)
dataset_train = dataset_train.map(formatting_prompts_func, batched=True)
dataset_ppl = dataset_ppl.map(formatting_prompts_func, batched=True)
print("\n dataset sample [0]")
print(dataset_train[0])
@ -146,8 +146,8 @@ if __name__ == "__main__":
model = FastLanguageModel.get_peft_model(
model,
r = 16,
target_modules = [
r=16,
target_modules=[
"k_proj",
"q_proj",
"v_proj",
@ -156,40 +156,40 @@ if __name__ == "__main__":
"down_proj",
"up_proj",
],
lora_alpha = 16,
lora_dropout = 0,
bias = "none",
use_gradient_checkpointing = "unsloth",
random_state = 3407,
use_rslora = False,
loftq_config = None,
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=3407,
use_rslora=False,
loftq_config=None,
)
from unsloth import is_bfloat16_supported
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = dataset_train,
dataset_text_field = "text",
max_seq_length = 2048,
data_collator = DataCollatorForSeq2Seq(tokenizer = tokenizer),
dataset_num_proc = 2,
packing = False,
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
warmup_ratio = 0.1,
max_steps = 200,
learning_rate = 2e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 50,
optim = "adamw_8bit",
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "outputs",
report_to = "none",
model=model,
tokenizer=tokenizer,
train_dataset=dataset_train,
dataset_text_field="text",
max_seq_length=2048,
data_collator=DataCollatorForSeq2Seq(tokenizer=tokenizer),
dataset_num_proc=2,
packing=False,
args=TrainingArguments(
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
warmup_ratio=0.1,
max_steps=200,
learning_rate=2e-4,
fp16=not is_bfloat16_supported(),
bf16=is_bfloat16_supported(),
logging_steps=50,
optim="adamw_8bit",
lr_scheduler_type="linear",
seed=3407,
output_dir="outputs",
report_to="none",
),
)
@ -197,8 +197,8 @@ if __name__ == "__main__":
trainer = train_on_responses_only(
trainer,
instruction_part = "<|start_header_id|>user<|end_header_id|>\n\n",
response_part = "<|start_header_id|>assistant<|end_header_id|>\n\n",
instruction_part="<|start_header_id|>user<|end_header_id|>\n\n",
response_part="<|start_header_id|>assistant<|end_header_id|>\n\n",
)
tokenizer.decode(trainer.train_dataset[0]["input_ids"])
@ -211,7 +211,7 @@ if __name__ == "__main__":
# saving and merging the model to local disk
print("merge and save to local disk")
model.save_pretrained_merged(
save_directory = "./unsloth_out/merged_llama_text_model", tokenizer = tokenizer
save_directory="./unsloth_out/merged_llama_text_model", tokenizer=tokenizer
)
# print("cleaning")
@ -223,10 +223,10 @@ if __name__ == "__main__":
# load model from local disk and test
print("Loading merged model in 4 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_llama_text_model",
max_seq_length = 2048,
load_in_4bit = True,
load_in_8bit = False,
model_name="./unsloth_out/merged_llama_text_model",
max_seq_length=2048,
load_in_4bit=True,
load_in_8bit=False,
)
add_to_comparison(
@ -235,7 +235,7 @@ if __name__ == "__main__":
print("Computing 8-bit model perplexity in subprocess...")
result_queue = mp.Queue()
p = mp.Process(target = load_and_compute_8bit_ppl, args = (result_queue, False, True))
p = mp.Process(target=load_and_compute_8bit_ppl, args=(result_queue, False, True))
p.start()
p.join()
@ -244,10 +244,10 @@ if __name__ == "__main__":
print("Loading merged model in 16 bit for perplexity test")
merged_model, merged_tokenizer = FastLanguageModel.from_pretrained(
model_name = "./unsloth_out/merged_llama_text_model",
max_seq_length = 2048,
load_in_4bit = False,
load_in_8bit = False,
model_name="./unsloth_out/merged_llama_text_model",
max_seq_length=2048,
load_in_4bit=False,
load_in_8bit=False,
)
add_to_comparison(

14
tests/saving/non_peft/test_mistral_non_peft.py
View file

@ -16,12 +16,12 @@ print("🔍 PHASE 1: Loading Base Model")
print(f"{'='*80}")
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/mistral-7b-v0.3",
max_seq_length = 2048,
dtype = None,
load_in_4bit = True,
load_in_8bit = False,
full_finetuning = False,
model_name="unsloth/mistral-7b-v0.3",
max_seq_length=2048,
dtype=None,
load_in_4bit=True,
load_in_8bit=False,
full_finetuning=False,
)
@ -34,7 +34,7 @@ print(f"\n{'='*80}")
print("🔍 PHASE 2: Attempting save_pretrained_merged (Should Warn)")
print(f"{'='*80}")
with warnings.catch_warnings(record = True) as w:
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
model.save_pretrained_merged("test_output", tokenizer)

14
tests/saving/non_peft/test_whisper_non_peft.py
View file

@ -16,12 +16,12 @@ print("🔍 PHASE 1: Loading Base Model")
print(f"{'='*80}")
model, tokenizer = FastModel.from_pretrained(
model_name = "unsloth/whisper-large-v3",
dtype = None, # Leave as None for auto detection
load_in_4bit = False, # Set to True to do 4bit quantization which reduces memory
auto_model = WhisperForConditionalGeneration,
whisper_language = "English",
whisper_task = "transcribe",
model_name="unsloth/whisper-large-v3",
dtype=None, # Leave as None for auto detection
load_in_4bit=False, # Set to True to do 4bit quantization which reduces memory
auto_model=WhisperForConditionalGeneration,
whisper_language="English",
whisper_task="transcribe",
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
)
@ -34,7 +34,7 @@ print(f"\n{'='*80}")
print("🔍 PHASE 2: Attempting save_pretrained_merged (Should Warn)")
print(f"{'='*80}")
with warnings.catch_warnings(record = True) as w:
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
model.save_pretrained_merged("test_output", tokenizer)

76
tests/saving/text_to_speech_models/test_orpheus.py
View file

@ -30,10 +30,10 @@ print(f"{'='*80}")
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/orpheus-3b-0.1-ft",
max_seq_length = 2048, # Choose any for long context!
dtype = None, # Select None for auto detection
load_in_4bit = False, # Select True for 4bit which reduces memory usage
model_name="unsloth/orpheus-3b-0.1-ft",
max_seq_length=2048, # Choose any for long context!
dtype=None, # Select None for auto detection
load_in_4bit=False, # Select True for 4bit which reduces memory usage
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
)
@ -42,8 +42,8 @@ base_model_class = model.__class__.__name__
model = FastLanguageModel.get_peft_model(
model,
r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
target_modules = [
r=64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
target_modules=[
"q_proj",
"k_proj",
"v_proj",
@ -52,14 +52,14 @@ model = FastLanguageModel.get_peft_model(
"up_proj",
"down_proj",
],
lora_alpha = 64,
lora_dropout = 0, # Supports any, but = 0 is optimized
bias = "none", # Supports any, but = "none" is optimized
lora_alpha=64,
lora_dropout=0, # Supports any, but = 0 is optimized
bias="none", # Supports any, but = "none" is optimized
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
random_state = 3407,
use_rslora = False, # We support rank stabilized LoRA
loftq_config = None, # And LoftQ
use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context
random_state=3407,
use_rslora=False, # We support rank stabilized LoRA
loftq_config=None, # And LoftQ
)
print("✅ Model and LoRA adapters loaded successfully!")
@ -112,10 +112,10 @@ print(f"{'='*80}")
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "unsloth/orpheus-3b-0.1-ft",
max_seq_length = 2048, # Choose any for long context!
dtype = None, # Select None for auto detection
load_in_4bit = False, # Select True for 4bit which reduces memory usage
model_name="unsloth/orpheus-3b-0.1-ft",
max_seq_length=2048, # Choose any for long context!
dtype=None, # Select None for auto detection
load_in_4bit=False, # Select True for 4bit which reduces memory usage
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
)
@ -148,18 +148,18 @@ prompts_ = [(f"{chosen_voice}: " + p) if chosen_voice else p for p in prompts]
all_input_ids = []
for prompt in prompts_:
input_ids = tokenizer(prompt, return_tensors = "pt").input_ids
input_ids = tokenizer(prompt, return_tensors="pt").input_ids
all_input_ids.append(input_ids)
start_token = torch.tensor([[128259]], dtype = torch.int64) # Start of human
start_token = torch.tensor([[128259]], dtype=torch.int64) # Start of human
end_tokens = torch.tensor(
[[128009, 128260]], dtype = torch.int64
[[128009, 128260]], dtype=torch.int64
) # End of text, End of human
all_modified_input_ids = []
for input_ids in all_input_ids:
modified_input_ids = torch.cat(
[start_token, input_ids, end_tokens], dim = 1
[start_token, input_ids, end_tokens], dim=1
) # SOH SOT Text EOT EOH
all_modified_input_ids.append(modified_input_ids)
@ -171,39 +171,39 @@ max_length = max(
for modified_input_ids in all_modified_input_ids:
padding = max_length - modified_input_ids.shape[1]
padded_tensor = torch.cat(
[torch.full((1, padding), 128263, dtype = torch.int64), modified_input_ids], dim = 1
[torch.full((1, padding), 128263, dtype=torch.int64), modified_input_ids], dim=1
)
attention_mask = torch.cat(
[
torch.zeros((1, padding), dtype = torch.int64),
torch.ones((1, modified_input_ids.shape[1]), dtype = torch.int64),
torch.zeros((1, padding), dtype=torch.int64),
torch.ones((1, modified_input_ids.shape[1]), dtype=torch.int64),
],
dim = 1,
dim=1,
)
all_padded_tensors.append(padded_tensor)
all_attention_masks.append(attention_mask)
all_padded_tensors = torch.cat(all_padded_tensors, dim = 0)
all_attention_masks = torch.cat(all_attention_masks, dim = 0)
all_padded_tensors = torch.cat(all_padded_tensors, dim=0)
all_attention_masks = torch.cat(all_attention_masks, dim=0)
input_ids = all_padded_tensors.to("cuda")
attention_mask = all_attention_masks.to("cuda")
generated_ids = model.generate(
input_ids = input_ids,
attention_mask = attention_mask,
max_new_tokens = 1200,
do_sample = True,
temperature = 0.6,
top_p = 0.95,
repetition_penalty = 1.1,
num_return_sequences = 1,
eos_token_id = 128258,
use_cache = True,
input_ids=input_ids,
attention_mask=attention_mask,
max_new_tokens=1200,
do_sample=True,
temperature=0.6,
top_p=0.95,
repetition_penalty=1.1,
num_return_sequences=1,
eos_token_id=128258,
use_cache=True,
)
token_to_find = 128257
token_to_remove = 128258
token_indices = (generated_ids == token_to_find).nonzero(as_tuple = True)
token_indices = (generated_ids == token_to_find).nonzero(as_tuple=True)
if len(token_indices[1]) > 0:
last_occurrence_idx = token_indices[1][-1].item()

110
tests/saving/vision_models/test_save_merge_qwen2.5vl32B_model_ocr_benchmark.py
View file

@ -22,7 +22,7 @@ from tests.utils.ocr_eval import OCRModelEvaluator
## Dataset Preparation
from datasets import load_dataset
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train")
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split="train")
# To select the first 2000 examples
train_dataset = dataset.select(range(2000))
@ -81,39 +81,39 @@ model_comparison_results = {}
# Load Base Model
model, tokenizer = FastVisionModel.from_pretrained(
model_name = "unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit",
max_seq_length = 2048, # Choose any for long context!
load_in_4bit = True, # 4 bit quantization to reduce memory
load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory
full_finetuning = False, # [NEW!] We have full finetuning now!
model_name="unsloth/Qwen2.5-VL-32B-Instruct-bnb-4bit",
max_seq_length=2048, # Choose any for long context!
load_in_4bit=True, # 4 bit quantization to reduce memory
load_in_8bit=False, # [NEW!] A bit more accurate, uses 2x memory
full_finetuning=False, # [NEW!] We have full finetuning now!
)
# benchmark base model performance
model_name = "Unsloth Base model"
FastVisionModel.for_inference(model)
avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model, tokenizer, eval_dataset, output_dir = "unsloth_base_model_results"
model, tokenizer, eval_dataset, output_dir="unsloth_base_model_results"
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
## Lora Finetuning
model = FastVisionModel.get_peft_model(
model,
finetune_vision_layers = True, # Turn off for just text!
finetune_language_layers = True, # Should leave on!
finetune_attention_modules = True, # Attention good for GRPO
finetune_mlp_modules = True, # SHould leave on always!
r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
finetune_vision_layers=True, # Turn off for just text!
finetune_language_layers=True, # Should leave on!
finetune_attention_modules=True, # Attention good for GRPO
finetune_mlp_modules=True, # SHould leave on always!
r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
# target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
# "gate_proj", "up_proj", "down_proj",],
lora_alpha = 32,
lora_dropout = 0, # Supports any, but = 0 is optimized
bias = "none", # Supports any, but = "none" is optimized
lora_alpha=32,
lora_dropout=0, # Supports any, but = 0 is optimized
bias="none", # Supports any, but = "none" is optimized
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
random_state = 3407,
use_rslora = False, # We support rank stabilized LoRA
loftq_config = None, # And LoftQ
use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context
random_state=3407,
use_rslora=False, # We support rank stabilized LoRA
loftq_config=None, # And LoftQ
)
from unsloth import is_bf16_supported
@ -124,40 +124,40 @@ model.config.use_cache = False
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
data_collator = UnslothVisionDataCollator(model, tokenizer),
train_dataset = train_dataset,
args = SFTConfig(
model=model,
tokenizer=tokenizer,
data_collator=UnslothVisionDataCollator(model, tokenizer),
train_dataset=train_dataset,
args=SFTConfig(
# per_device_train_batch_size = 4,
# gradient_accumulation_steps = 8,
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
gradient_checkpointing = True,
gradient_checkpointing_kwargs = {
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
gradient_checkpointing=True,
gradient_checkpointing_kwargs={
"use_reentrant": False
}, # use reentrant checkpointing
max_grad_norm = 0.3, # max gradient norm based on QLoRA paper
warmup_ratio = 0.03,
max_grad_norm=0.3, # max gradient norm based on QLoRA paper
warmup_ratio=0.03,
# num_train_epochs = 2, # Set this instead of max_steps for full training runs
max_steps = 60,
learning_rate = 2e-4,
fp16 = not is_bf16_supported(),
bf16 = is_bf16_supported(),
logging_steps = 5,
save_strategy = "epoch",
optim = "adamw_torch_fused",
weight_decay = 0.01,
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "unsloth-qwen2.5-vl-32b-french-ocr-checkpoints",
report_to = "none", # For Weights and Biases
max_steps=60,
learning_rate=2e-4,
fp16=not is_bf16_supported(),
bf16=is_bf16_supported(),
logging_steps=5,
save_strategy="epoch",
optim="adamw_torch_fused",
weight_decay=0.01,
lr_scheduler_type="linear",
seed=3407,
output_dir="unsloth-qwen2.5-vl-32b-french-ocr-checkpoints",
report_to="none", # For Weights and Biases
# You MUST put the below items for vision finetuning:
remove_unused_columns = False,
dataset_text_field = "",
dataset_kwargs = {"skip_prepare_dataset": True},
dataset_num_proc = 4,
max_seq_length = 2048,
remove_unused_columns=False,
dataset_text_field="",
dataset_kwargs={"skip_prepare_dataset": True},
dataset_num_proc=4,
max_seq_length=2048,
),
)
@ -173,7 +173,7 @@ tokenizer.save_pretrained("unsloth-qwen2.5-vl-32b-french-ocr-adapter")
model_name = "Unsloth lora adapter model"
FastVisionModel.for_inference(model)
avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model, tokenizer, eval_dataset, output_dir = "unsloth_lora_model_results"
model, tokenizer, eval_dataset, output_dir="unsloth_lora_model_results"
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
@ -195,7 +195,7 @@ print((base.__class__.__name__))
# merge default 16 bits
model.save_pretrained_merged(
save_directory = "qwen2.5-ocr-merged-finetune-merge-16bit", tokenizer = tokenizer
save_directory="qwen2.5-ocr-merged-finetune-merge-16bit", tokenizer=tokenizer
)
@ -204,7 +204,7 @@ model.save_pretrained_merged(
### 16 bits merged model
model, tokenizer = FastVisionModel.from_pretrained(
"./qwen2.5-ocr-merged-finetune-merge-16bit", load_in_4bit = False, load_in_8bit = False
"./qwen2.5-ocr-merged-finetune-merge-16bit", load_in_4bit=False, load_in_8bit=False
)
# benchmark 4bit loaded, 16bits merged model performance
@ -215,13 +215,13 @@ avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model,
tokenizer,
eval_dataset,
output_dir = "unsloth_16bits_merged_model_load_16bits_results",
output_dir="unsloth_16bits_merged_model_load_16bits_results",
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
# load 16bits-merged model in 4 bits
model, tokenizer = FastVisionModel.from_pretrained(
"./qwen2.5-ocr-merged-finetune-merge-16bit", load_in_4bit = True, load_in_8bit = False
"./qwen2.5-ocr-merged-finetune-merge-16bit", load_in_4bit=True, load_in_8bit=False
)
# benchmark 4bit loaded, 16bits merged model performance
@ -232,13 +232,13 @@ avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model,
tokenizer,
eval_dataset,
output_dir = "unsloth_16bits_merged_model_load_4bits_results",
output_dir="unsloth_16bits_merged_model_load_4bits_results",
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
# load model in 8 bits
model, tokenizer = FastVisionModel.from_pretrained(
"./qwen2.5-ocr-merged-finetune-merge-16bit", load_in_4bit = False, load_in_8bit = True
"./qwen2.5-ocr-merged-finetune-merge-16bit", load_in_4bit=False, load_in_8bit=True
)
# benchmark 4bit loaded, 16bits merged model performance
@ -247,7 +247,7 @@ avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model,
tokenizer,
eval_dataset,
output_dir = "unsloth_16bits_merged_model_load_8bits_results",
output_dir="unsloth_16bits_merged_model_load_8bits_results",
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)

110
tests/saving/vision_models/test_save_merge_vision_model_ocr_benchmark.py
View file

@ -22,7 +22,7 @@ from tests.utils.ocr_eval import OCRModelEvaluator
## Dataset Preparation
from datasets import load_dataset
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split = "train")
dataset = load_dataset("lbourdois/OCR-liboaccn-OPUS-MIT-5M-clean", "en", split="train")
# To select the first 2000 examples
train_dataset = dataset.select(range(2000))
@ -81,39 +81,39 @@ model_comparison_results = {}
# Load Base Model
model, tokenizer = FastVisionModel.from_pretrained(
model_name = "unsloth/Qwen2-VL-7B-Instruct",
max_seq_length = 2048, # Choose any for long context!
load_in_4bit = True, # 4 bit quantization to reduce memory
load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory
full_finetuning = False, # [NEW!] We have full finetuning now!
model_name="unsloth/Qwen2-VL-7B-Instruct",
max_seq_length=2048, # Choose any for long context!
load_in_4bit=True, # 4 bit quantization to reduce memory
load_in_8bit=False, # [NEW!] A bit more accurate, uses 2x memory
full_finetuning=False, # [NEW!] We have full finetuning now!
)
# benchmark base model performance
model_name = "Unsloth Base model"
FastVisionModel.for_inference(model)
avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model, tokenizer, eval_dataset, output_dir = "unsloth_base_model_results"
model, tokenizer, eval_dataset, output_dir="unsloth_base_model_results"
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
## Lora Finetuning
model = FastVisionModel.get_peft_model(
model,
finetune_vision_layers = True, # Turn off for just text!
finetune_language_layers = True, # Should leave on!
finetune_attention_modules = True, # Attention good for GRPO
finetune_mlp_modules = True, # SHould leave on always!
r = 16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
finetune_vision_layers=True, # Turn off for just text!
finetune_language_layers=True, # Should leave on!
finetune_attention_modules=True, # Attention good for GRPO
finetune_mlp_modules=True, # SHould leave on always!
r=16, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
# target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
# "gate_proj", "up_proj", "down_proj",],
lora_alpha = 32,
lora_dropout = 0, # Supports any, but = 0 is optimized
bias = "none", # Supports any, but = "none" is optimized
lora_alpha=32,
lora_dropout=0, # Supports any, but = 0 is optimized
bias="none", # Supports any, but = "none" is optimized
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
random_state = 3407,
use_rslora = False, # We support rank stabilized LoRA
loftq_config = None, # And LoftQ
use_gradient_checkpointing="unsloth", # True or "unsloth" for very long context
random_state=3407,
use_rslora=False, # We support rank stabilized LoRA
loftq_config=None, # And LoftQ
)
from unsloth import is_bf16_supported
@ -124,40 +124,40 @@ model.config.use_cache = False
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
data_collator = UnslothVisionDataCollator(model, tokenizer),
train_dataset = train_dataset,
args = SFTConfig(
model=model,
tokenizer=tokenizer,
data_collator=UnslothVisionDataCollator(model, tokenizer),
train_dataset=train_dataset,
args=SFTConfig(
# per_device_train_batch_size = 4,
# gradient_accumulation_steps = 8,
per_device_train_batch_size = 2,
gradient_accumulation_steps = 4,
gradient_checkpointing = True,
gradient_checkpointing_kwargs = {
per_device_train_batch_size=2,
gradient_accumulation_steps=4,
gradient_checkpointing=True,
gradient_checkpointing_kwargs={
"use_reentrant": False
}, # use reentrant checkpointing
max_grad_norm = 0.3, # max gradient norm based on QLoRA paper
warmup_ratio = 0.03,
max_grad_norm=0.3, # max gradient norm based on QLoRA paper
warmup_ratio=0.03,
# num_train_epochs = 2, # Set this instead of max_steps for full training runs
max_steps = 60,
learning_rate = 2e-4,
fp16 = not is_bf16_supported(),
bf16 = is_bf16_supported(),
logging_steps = 5,
save_strategy = "epoch",
optim = "adamw_torch_fused",
weight_decay = 0.01,
lr_scheduler_type = "linear",
seed = 3407,
output_dir = "unsloth-qwen2-7vl-french-ocr-checkpoints",
report_to = "none", # For Weights and Biases
max_steps=60,
learning_rate=2e-4,
fp16=not is_bf16_supported(),
bf16=is_bf16_supported(),
logging_steps=5,
save_strategy="epoch",
optim="adamw_torch_fused",
weight_decay=0.01,
lr_scheduler_type="linear",
seed=3407,
output_dir="unsloth-qwen2-7vl-french-ocr-checkpoints",
report_to="none", # For Weights and Biases
# You MUST put the below items for vision finetuning:
remove_unused_columns = False,
dataset_text_field = "",
dataset_kwargs = {"skip_prepare_dataset": True},
dataset_num_proc = 4,
max_seq_length = 2048,
remove_unused_columns=False,
dataset_text_field="",
dataset_kwargs={"skip_prepare_dataset": True},
dataset_num_proc=4,
max_seq_length=2048,
),
)
@ -173,7 +173,7 @@ tokenizer.save_pretrained("unsloth-qwen2-7vl-french-ocr-adapter")
model_name = "Unsloth lora adapter model"
FastVisionModel.for_inference(model)
avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model, tokenizer, eval_dataset, output_dir = "unsloth_lora_model_results"
model, tokenizer, eval_dataset, output_dir="unsloth_lora_model_results"
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
@ -195,7 +195,7 @@ print((base.__class__.__name__))
# merge default 16 bits
model.save_pretrained_merged(
save_directory = "qwen2-ocr-merged-finetune-merge-16bit", tokenizer = tokenizer
save_directory="qwen2-ocr-merged-finetune-merge-16bit", tokenizer=tokenizer
)
@ -204,7 +204,7 @@ model.save_pretrained_merged(
### 16 bits merged model
model, tokenizer = FastVisionModel.from_pretrained(
"./qwen2-ocr-merged-finetune-merge-16bit", load_in_4bit = False, load_in_8bit = False
"./qwen2-ocr-merged-finetune-merge-16bit", load_in_4bit=False, load_in_8bit=False
)
# benchmark 4bit loaded, 16bits merged model performance
@ -215,13 +215,13 @@ avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model,
tokenizer,
eval_dataset,
output_dir = "unsloth_16bits_merged_model_load_16bits_results",
output_dir="unsloth_16bits_merged_model_load_16bits_results",
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
# load 16bits-merged model in 4 bits
model, tokenizer = FastVisionModel.from_pretrained(
"./qwen2-ocr-merged-finetune-merge-16bit", load_in_4bit = True, load_in_8bit = False
"./qwen2-ocr-merged-finetune-merge-16bit", load_in_4bit=True, load_in_8bit=False
)
# benchmark 4bit loaded, 16bits merged model performance
@ -232,13 +232,13 @@ avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model,
tokenizer,
eval_dataset,
output_dir = "unsloth_16bits_merged_model_load_4bits_results",
output_dir="unsloth_16bits_merged_model_load_4bits_results",
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)
# load model in 8 bits
model, tokenizer = FastVisionModel.from_pretrained(
"./qwen2-ocr-merged-finetune-merge-16bit", load_in_4bit = False, load_in_8bit = True
"./qwen2-ocr-merged-finetune-merge-16bit", load_in_4bit=False, load_in_8bit=True
)
# benchmark 4bit loaded, 16bits merged model performance
@ -247,7 +247,7 @@ avg_wer, avg_cer = ocr_evaluator.evaluate_model(
model,
tokenizer,
eval_dataset,
output_dir = "unsloth_16bits_merged_model_load_8bits_results",
output_dir="unsloth_16bits_merged_model_load_8bits_results",
)
ocr_evaluator.add_to_comparison(model_name, avg_wer, avg_cer)