Merge branch 'main' into nightly
This commit is contained in:
Daniel Han
commit
8ba4f3f808
36 changed files with 1313 additions and 95 deletions
3
.github/ISSUE_TEMPLATE/bug---issue.md
vendored
3
.github/ISSUE_TEMPLATE/bug---issue.md
vendored
|
|
@ -15,5 +15,4 @@ assignees: ''
|
|||
6. Which trainer? `SFTTrainer`, `GRPOTrainer` etc
|
||||
7. **Minimal code to reproduce error Remove Hugging Face token!**
|
||||
|
||||
You can also join our Discord: https://discord.com/invite/unsloth
|
||||
Have you tried visiting our Docs? https://docs.unsloth.ai/basics/errors-troubleshooting
|
||||
🦥 You can also ask via our Reddit page: https://www.reddit.com/r/unsloth/
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ dynamic = ["version"]
|
|||
description = "2-5X faster LLM finetuning"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9,<3.13"
|
||||
license = "Apache-2.0"
|
||||
license = {text = "Apache-2.0"}
|
||||
keywords = ["ai", "llm",]
|
||||
authors = [
|
||||
{email = "info@unsloth.ai"},
|
||||
|
|
@ -37,7 +37,7 @@ triton = [
|
|||
]
|
||||
|
||||
huggingface = [
|
||||
"unsloth_zoo>=2025.6.1",
|
||||
"unsloth_zoo>=2025.6.3",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2",
|
||||
|
|
@ -50,7 +50,7 @@ huggingface = [
|
|||
"accelerate>=0.34.1",
|
||||
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0",
|
||||
"peft>=0.7.1,!=0.11.0",
|
||||
"protobuf<4.0.0",
|
||||
"protobuf",
|
||||
"huggingface_hub",
|
||||
"hf_transfer",
|
||||
"unsloth[triton]",
|
||||
|
|
@ -381,7 +381,7 @@ colab-ampere-torch220 = [
|
|||
"flash-attn>=2.6.3",
|
||||
]
|
||||
colab-new = [
|
||||
"unsloth_zoo>=2025.5.11",
|
||||
"unsloth_zoo>=2025.6.3",
|
||||
"packaging",
|
||||
"tyro",
|
||||
"transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2",
|
||||
|
|
@ -391,7 +391,7 @@ colab-new = [
|
|||
"psutil",
|
||||
"wheel>=0.42.0",
|
||||
"numpy",
|
||||
"protobuf<4.0.0",
|
||||
"protobuf",
|
||||
"huggingface_hub",
|
||||
"hf_transfer",
|
||||
"bitsandbytes>=0.45.5",
|
||||
|
|
@ -403,7 +403,7 @@ colab-no-deps = [
|
|||
"peft>=0.7.1",
|
||||
"xformers",
|
||||
"bitsandbytes>=0.45.5",
|
||||
"protobuf<4.0.0",
|
||||
"protobuf",
|
||||
]
|
||||
colab = [
|
||||
"unsloth[cu121]",
|
||||
|
|
@ -607,6 +607,21 @@ intel-gpu-torch260 = [
|
|||
"torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c4c5c67625cdacf35765c2b94e61fe166e3c3f4a14521b1212a59ad1b3eb0f2e ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'",
|
||||
"torch @ https://download.pytorch.org/whl/xpu/torch-2.6.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=e6864f7a60a5ecc43d5d38f59a16e5dd132384f73dfd3a697f74944026038f7b ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'",
|
||||
]
|
||||
intel-gpu-torch270 = [
|
||||
"unsloth[huggingface]",
|
||||
|
||||
"pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=749a7098492c6a27b356c97149a4a62973b953eae60bc1b6259260974f344913 ; platform_system == 'Linux' and python_version == '3.9' and platform_machine == 'x86_64'",
|
||||
"pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=44362e80abd752471a08341093321955b066daa2cfb4810e73b8e3b240850f93 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'",
|
||||
"pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=faa6b8c945a837a080f641bc8ccc77a98fa66980dcd7e62e715fd853737343fd ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'",
|
||||
"pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=40f6fb65b345dc9a61813abe7ac9a585f2c9808f414d140cc2a5f11f53ee063c ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'",
|
||||
"pytorch_triton_xpu @ https://download.pytorch.org/whl/pytorch_triton_xpu-3.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=9821fe059de58e827ffc6aa10d69369b16c2f8c2a988b86bef9c2c6e396ab3aa ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'",
|
||||
|
||||
"torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp39-cp39-linux_x86_64.whl#sha256=f8ee75e50fcbb37ed5b498299ca2264da99ab278a93fae2358e921e4a6e28273 ; platform_system == 'Linux' and python_version == '3.9' and platform_machine == 'x86_64'",
|
||||
"torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp310-cp310-linux_x86_64.whl#sha256=d6fdc342961d98fdcd9d03dfd491a3208bb5f7fbb435841f8f72ce9fdcd2d026 ; platform_system == 'Linux' and python_version == '3.10' and platform_machine == 'x86_64'",
|
||||
"torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp311-cp311-linux_x86_64.whl#sha256=74d07f9357df5cf2bf223ad3c84de16346bfaa0504f988fdd5590d3e177e5e86 ; platform_system == 'Linux' and python_version == '3.11' and platform_machine == 'x86_64'",
|
||||
"torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp312-cp312-linux_x86_64.whl#sha256=c806d44aa2ca5d225629f6fbc6c994d5deaac2d2cde449195bc8e3522ddd219a ; platform_system == 'Linux' and python_version == '3.12' and platform_machine == 'x86_64'",
|
||||
"torch @ https://download.pytorch.org/whl/xpu/torch-2.7.0%2Bxpu-cp313-cp313-linux_x86_64.whl#sha256=25d8277b7f01d42e2e014ccbab57a2692b6ec4eff8dcf894eda1b297407cf97a ; platform_system == 'Linux' and python_version == '3.13' and platform_machine == 'x86_64'",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
homepage = "http://www.unsloth.ai"
|
||||
|
|
|
|||
|
|
@ -14,8 +14,10 @@ import gc
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.perplexity_eval import ppl_model, add_to_comparison, print_model_comparison
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ import gc
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.perplexity_eval import ppl_model, add_to_comparison, print_model_comparison
|
||||
|
|
|
|||
|
|
@ -14,8 +14,10 @@ import gc
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.perplexity_eval import ppl_model, add_to_comparison, print_model_comparison
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ import gc
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.perplexity_eval import ppl_model, add_to_comparison, print_model_comparison
|
||||
|
|
|
|||
|
|
@ -14,8 +14,9 @@ import gc
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.perplexity_eval import ppl_model, add_to_comparison, print_model_comparison
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ from huggingface_hub import HfFileSystem, hf_hub_download
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.perplexity_eval import ppl_model, add_to_comparison, print_model_comparison
|
||||
|
|
|
|||
|
|
@ -15,8 +15,9 @@ from huggingface_hub import HfFileSystem, hf_hub_download
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.perplexity_eval import ppl_model, add_to_comparison, print_model_comparison
|
||||
|
|
|
|||
|
|
@ -12,8 +12,9 @@ from pathlib import Path
|
|||
import multiprocessing as mp
|
||||
import gc
|
||||
from multiprocessing import Queue
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.aime_eval import evaluate_model_aime, compare_aime_results
|
||||
|
|
|
|||
67
tests/saving/non_peft/test_mistral_non_peft.py
Normal file
67
tests/saving/non_peft/test_mistral_non_peft.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
from unsloth import FastLanguageModel
|
||||
from transformers import AutoModelForCausalLM
|
||||
from peft import PeftModel
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 PHASE 1: Loading Base Model")
|
||||
print(f"{'='*80}")
|
||||
|
||||
model, tokenizer = FastLanguageModel.from_pretrained(
|
||||
model_name="unsloth/mistral-7b-v0.3",
|
||||
max_seq_length=2048,
|
||||
dtype=None,
|
||||
load_in_4bit=True,
|
||||
load_in_8bit=False,
|
||||
full_finetuning=False,
|
||||
)
|
||||
|
||||
|
||||
print("✅ Base model loaded successfully!")
|
||||
|
||||
### Attemtping save merge
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 PHASE 2: Attempting save_pretrained_merged (Should Warn)")
|
||||
print(f"{'='*80}")
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter("always")
|
||||
model.save_pretrained_merged("test_output", tokenizer)
|
||||
|
||||
# Verify warning
|
||||
assert len(w) >= 1, "Expected warning but none raised"
|
||||
warning_msg = str(w[0].message)
|
||||
expected_msg = "Model is not a PeftModel (no Lora adapters detected). Skipping Merge. Please use save_pretrained() or push_to_hub() instead!"
|
||||
assert expected_msg in warning_msg, f"Unexpected warning: {warning_msg}"
|
||||
assert expected_msg in warning_msg, f"Unexpected warning: {warning_msg}"
|
||||
|
||||
print("✅ Correct warning detected for non-PeftModel merge attempt!")
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 PHASE 3: Using save_pretrained (Should Succeed)")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error") # Treat warnings as errors here
|
||||
model.save_pretrained("test_output")
|
||||
print("✅ Standard save_pretrained completed successfully!")
|
||||
except Exception as e:
|
||||
assert False, f"Phase 3 failed: {e}"
|
||||
|
||||
safe_remove_directory("./test_output")
|
||||
safe_remove_directory("./unsloth_compiled_cache")
|
||||
67
tests/saving/non_peft/test_whisper_non_peft.py
Normal file
67
tests/saving/non_peft/test_whisper_non_peft.py
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
from unsloth import FastLanguageModel, FastModel
|
||||
from transformers import AutoModelForCausalLM, WhisperForConditionalGeneration
|
||||
from peft import PeftModel
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 PHASE 1: Loading Base Model")
|
||||
print(f"{'='*80}")
|
||||
|
||||
model, tokenizer = FastModel.from_pretrained(
|
||||
model_name = "unsloth/whisper-large-v3",
|
||||
dtype = None, # Leave as None for auto detection
|
||||
load_in_4bit = False, # Set to True to do 4bit quantization which reduces memory
|
||||
auto_model = WhisperForConditionalGeneration,
|
||||
whisper_language = "English",
|
||||
whisper_task = "transcribe",
|
||||
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
|
||||
)
|
||||
|
||||
print("✅ Base model loaded successfully!")
|
||||
|
||||
### Attemtping save merge
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 PHASE 2: Attempting save_pretrained_merged (Should Warn)")
|
||||
print(f"{'='*80}")
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter("always")
|
||||
model.save_pretrained_merged("test_output", tokenizer)
|
||||
|
||||
# Verify warning
|
||||
assert len(w) >= 1, "Expected warning but none raised"
|
||||
warning_msg = str(w[0].message)
|
||||
expected_msg = "Model is not a PeftModel (no Lora adapters detected). Skipping Merge. Please use save_pretrained() or push_to_hub() instead!"
|
||||
assert expected_msg in warning_msg, f"Unexpected warning: {warning_msg}"
|
||||
assert expected_msg in warning_msg, f"Unexpected warning: {warning_msg}"
|
||||
|
||||
print("✅ Correct warning detected for non-PeftModel merge attempt!")
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 PHASE 3: Using save_pretrained (Should Succeed)")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error") # Treat warnings as errors here
|
||||
model.save_pretrained("test_output")
|
||||
print("✅ Standard save_pretrained completed successfully!")
|
||||
except Exception as e:
|
||||
assert False, f"Phase 3 failed: {e}"
|
||||
|
||||
safe_remove_directory("./test_output")
|
||||
safe_remove_directory("./unsloth_compiled_cache")
|
||||
156
tests/saving/text_to_speech_models/test_csm.py
Normal file
156
tests/saving/text_to_speech_models/test_csm.py
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
from unsloth import FastLanguageModel, FastModel
|
||||
from transformers import CsmForConditionalGeneration
|
||||
import torch
|
||||
# ruff: noqa
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from peft import PeftModel
|
||||
import warnings
|
||||
import requests
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.os_utils import require_package, require_python_package
|
||||
|
||||
require_package("ffmpeg", "ffmpeg")
|
||||
require_python_package("soundfile")
|
||||
|
||||
import soundfile as sf
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 1: Loading Model and LoRA Adapters")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
model, tokenizer = FastModel.from_pretrained(
|
||||
model_name = "unsloth/csm-1b",
|
||||
max_seq_length= 2048, # Choose any for long context!
|
||||
dtype = None, # Leave as None for auto-detection
|
||||
auto_model = CsmForConditionalGeneration,
|
||||
load_in_4bit = False, # Select True for 4bit - reduces memory usage
|
||||
)
|
||||
|
||||
|
||||
base_model_class = model.__class__.__name__
|
||||
|
||||
|
||||
model = FastModel.get_peft_model(
|
||||
model,
|
||||
r = 32, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
|
||||
target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
|
||||
"gate_proj", "up_proj", "down_proj",],
|
||||
lora_alpha = 32,
|
||||
lora_dropout = 0, # Supports any, but = 0 is optimized
|
||||
bias = "none", # Supports any, but = "none" is optimized
|
||||
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
|
||||
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
|
||||
random_state = 3407,
|
||||
use_rslora = False, # We support rank stabilized LoRA
|
||||
loftq_config = None, # And LoftQ
|
||||
)
|
||||
|
||||
print("✅ Model and LoRA adapters loaded successfully!")
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 2: Checking Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
assert isinstance(model, PeftModel), "Model should be an instance of PeftModel"
|
||||
print("✅ Model is an instance of PeftModel!")
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 3: Checking Config Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
def find_lora_base_model(model_to_inspect):
|
||||
current = model_to_inspect
|
||||
if hasattr(current, "base_model"):
|
||||
current = current.base_model
|
||||
if hasattr(current, "model"):
|
||||
current = current.model
|
||||
return current
|
||||
pass
|
||||
|
||||
|
||||
config_model = find_lora_base_model(model) if isinstance(model, PeftModel) else model
|
||||
|
||||
assert config_model.__class__.__name__ == base_model_class, f"Expected config_model class to be {base_model_class}"
|
||||
print("✅ config_model returns correct Base Model class:", str(base_model_class))
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 4: Saving and Merging Model")
|
||||
print(f"{'='*80}")
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error") # Treat warnings as errors
|
||||
try:
|
||||
model.save_pretrained_merged("csm", tokenizer)
|
||||
print("✅ Model saved and merged successfully without warnings!")
|
||||
except Exception as e:
|
||||
assert False, f"Model saving/merging failed with exception: {e}"
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 5: Loading Model for Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
model, processor = FastModel.from_pretrained(
|
||||
model_name = "./csm",
|
||||
max_seq_length= 2048, # Choose any for long context!
|
||||
dtype = None, # Leave as None for auto-detection
|
||||
auto_model = CsmForConditionalGeneration,
|
||||
load_in_4bit = False, # Select True for 4bit - reduces memory usage
|
||||
)
|
||||
|
||||
from transformers import AutoProcessor
|
||||
processor = AutoProcessor.from_pretrained("unsloth/csm-1b")
|
||||
|
||||
print("✅ Model loaded for inference successfully!")
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 6: Running Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
from transformers import pipeline
|
||||
import torch
|
||||
output_audio_path = "csm_audio.wav"
|
||||
try:
|
||||
text = "We just finished fine tuning a text to speech model... and it's pretty good!"
|
||||
speaker_id = 0
|
||||
inputs = processor(f"[{speaker_id}]{text}", add_special_tokens=True).to("cuda")
|
||||
audio_values = model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=125, # 125 tokens is 10 seconds of audio, for longer speech increase this
|
||||
# play with these parameters to get the best results
|
||||
depth_decoder_temperature=0.6,
|
||||
depth_decoder_top_k=0,
|
||||
depth_decoder_top_p=0.9,
|
||||
temperature=0.8,
|
||||
top_k=50,
|
||||
top_p=1.0,
|
||||
#########################################################
|
||||
output_audio=True
|
||||
)
|
||||
audio = audio_values[0].to(torch.float32).cpu().numpy()
|
||||
sf.write("example_without_context.wav", audio, 24000)
|
||||
print(f"✅ Audio generated and saved to {output_audio_path}!")
|
||||
except Exception as e:
|
||||
assert False, f"Inference failed with exception: {e}"
|
||||
|
||||
|
||||
## assert that transcribed_text contains The birch canoe slid on the smooth planks. Glued the sheet to the dark blue background. It's easy to tell the depth of a well. Four hours of steady work faced us.
|
||||
|
||||
print("✅ All sections passed successfully!")
|
||||
|
||||
|
||||
safe_remove_directory("./unsloth_compiled_cache")
|
||||
safe_remove_directory("./csm")
|
||||
217
tests/saving/text_to_speech_models/test_lasa.py
Normal file
217
tests/saving/text_to_speech_models/test_lasa.py
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
from unsloth import FastLanguageModel, FastModel
|
||||
from transformers import CsmForConditionalGeneration
|
||||
import torch
|
||||
# ruff: noqa
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from peft import PeftModel
|
||||
import warnings
|
||||
import requests
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.os_utils import require_package, require_python_package
|
||||
|
||||
require_package("ffmpeg", "ffmpeg")
|
||||
require_python_package("soundfile")
|
||||
require_python_package("xcodec2")
|
||||
|
||||
import soundfile as sf
|
||||
from xcodec2.modeling_xcodec2 import XCodec2Model
|
||||
XCODEC2_MODEL_NAME = "HKUST-Audio/xcodec2"
|
||||
SAMPLE_RATE = 16000
|
||||
DEVICE = "cuda"
|
||||
|
||||
try:
|
||||
codec_model = XCodec2Model.from_pretrained(XCODEC2_MODEL_NAME)
|
||||
|
||||
except Exception as e:
|
||||
raise f"ERROR loading XCodec2 model: {e}."
|
||||
|
||||
codec_model.to('cpu')
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 1: Loading Model and LoRA Adapters")
|
||||
print(f"{'='*80}")
|
||||
|
||||
max_seq_length = 2048
|
||||
model, tokenizer = FastLanguageModel.from_pretrained(
|
||||
model_name = "unsloth/Llasa-1B",
|
||||
max_seq_length = max_seq_length,
|
||||
dtype = None, # Select None for auto detection
|
||||
load_in_4bit = False, # Choose True for 4bit which reduces memory
|
||||
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
|
||||
)
|
||||
|
||||
base_model_class = model.__class__.__name__
|
||||
|
||||
|
||||
model = FastLanguageModel.get_peft_model(
|
||||
model,
|
||||
r = 128, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
|
||||
target_modules = ["q_proj", "v_proj"],
|
||||
lora_alpha = 128,
|
||||
lora_dropout = 0, # Supports any, but = 0 is optimized
|
||||
bias = "none", # Supports any, but = "none" is optimized
|
||||
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
|
||||
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
|
||||
random_state = 3407,
|
||||
use_rslora = False, # We support rank stabilized LoRA
|
||||
loftq_config = None, # And LoftQ
|
||||
)
|
||||
|
||||
print("✅ Model and LoRA adapters loaded successfully!")
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 2: Checking Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
assert isinstance(model, PeftModel), "Model should be an instance of PeftModel"
|
||||
print("✅ Model is an instance of PeftModel!")
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 3: Checking Config Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
def find_lora_base_model(model_to_inspect):
|
||||
current = model_to_inspect
|
||||
if hasattr(current, "base_model"):
|
||||
current = current.base_model
|
||||
if hasattr(current, "model"):
|
||||
current = current.model
|
||||
return current
|
||||
pass
|
||||
|
||||
|
||||
config_model = find_lora_base_model(model) if isinstance(model, PeftModel) else model
|
||||
|
||||
assert config_model.__class__.__name__ == base_model_class, f"Expected config_model class to be {base_model_class}"
|
||||
print("✅ config_model returns correct Base Model class:", str(base_model_class))
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 4: Saving and Merging Model")
|
||||
print(f"{'='*80}")
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error") # Treat warnings as errors
|
||||
try:
|
||||
model.save_pretrained_merged("lasa", tokenizer)
|
||||
print("✅ Model saved and merged successfully without warnings!")
|
||||
except Exception as e:
|
||||
assert False, f"Model saving/merging failed with exception: {e}"
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 5: Loading Model for Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
model, tokenizer = FastLanguageModel.from_pretrained(
|
||||
model_name = "./lasa",
|
||||
max_seq_length = max_seq_length,
|
||||
dtype = None, # Select None for auto detection
|
||||
load_in_4bit = False, # Choose True for 4bit which reduces memory
|
||||
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
|
||||
)
|
||||
|
||||
#from transformers import AutoProcessor
|
||||
#processor = AutoProcessor.from_pretrained("unsloth/csm-1b")
|
||||
|
||||
print("✅ Model loaded for inference successfully!")
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 6: Running Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
from transformers import pipeline
|
||||
import torch
|
||||
output_audio_path = "lasa_audio.wav"
|
||||
input_text = "Hey there my name is Elise, <giggles> and I'm a speech generation model that can sound like a person."
|
||||
|
||||
FastLanguageModel.for_inference(model)
|
||||
|
||||
def ids_to_speech_tokens(speech_ids):
|
||||
|
||||
speech_tokens_str = []
|
||||
for speech_id in speech_ids:
|
||||
speech_tokens_str.append(f"<|s_{speech_id}|>")
|
||||
return speech_tokens_str
|
||||
|
||||
def extract_speech_ids(speech_tokens_str):
|
||||
|
||||
speech_ids = []
|
||||
for token_str in speech_tokens_str:
|
||||
if token_str.startswith('<|s_') and token_str.endswith('|>'):
|
||||
num_str = token_str[4:-2]
|
||||
|
||||
num = int(num_str)
|
||||
speech_ids.append(num)
|
||||
else:
|
||||
print(f"Unexpected token: {token_str}")
|
||||
return speech_ids
|
||||
|
||||
#TTS start!
|
||||
with torch.inference_mode():
|
||||
with torch.amp.autocast('cuda',dtype=model.dtype):
|
||||
formatted_text = f"<|TEXT_UNDERSTANDING_START|>{input_text}<|TEXT_UNDERSTANDING_END|>"
|
||||
|
||||
# Tokenize the text
|
||||
chat = [
|
||||
{"role": "user", "content": "Convert the text to speech:" + formatted_text},
|
||||
{"role": "assistant", "content": "<|SPEECH_GENERATION_START|>"}
|
||||
]
|
||||
|
||||
input_ids = tokenizer.apply_chat_template(
|
||||
chat,
|
||||
tokenize=True,
|
||||
return_tensors='pt',
|
||||
continue_final_message=True
|
||||
)
|
||||
input_ids = input_ids.to('cuda')
|
||||
|
||||
speech_end_id = tokenizer.convert_tokens_to_ids('<|SPEECH_GENERATION_END|>')
|
||||
|
||||
# Generate the speech autoregressively
|
||||
outputs = model.generate(
|
||||
input_ids,
|
||||
max_length=2048, # We trained our model with a max length of 2048
|
||||
eos_token_id= speech_end_id ,
|
||||
do_sample=True,
|
||||
top_p=1.2, # Adjusts the diversity of generated content
|
||||
temperature=1.2, # Controls randomness in output
|
||||
)
|
||||
# Extract the speech tokens
|
||||
generated_ids = outputs[0][input_ids.shape[1]:-1]
|
||||
|
||||
speech_tokens = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
|
||||
|
||||
# Convert token <|s_23456|> to int 23456
|
||||
speech_tokens = extract_speech_ids(speech_tokens)
|
||||
|
||||
speech_tokens = torch.tensor(speech_tokens).cpu().unsqueeze(0).unsqueeze(0)
|
||||
|
||||
# Decode the speech tokens to speech waveform
|
||||
gen_wav = codec_model.decode_code(speech_tokens)
|
||||
try:
|
||||
sf.write(output_audio_path, gen_wav[0, 0, :].cpu().numpy(), 16000)
|
||||
except Exception as e:
|
||||
assert False, f"Inference failed with exception: {e}"
|
||||
|
||||
|
||||
## assert that transcribed_text contains The birch canoe slid on the smooth planks. Glued the sheet to the dark blue background. It's easy to tell the depth of a well. Four hours of steady work faced us.
|
||||
|
||||
print("✅ All sections passed successfully!")
|
||||
|
||||
|
||||
safe_remove_directory("./unsloth_compiled_cache")
|
||||
safe_remove_directory("./lasa")
|
||||
254
tests/saving/text_to_speech_models/test_orpheus.py
Normal file
254
tests/saving/text_to_speech_models/test_orpheus.py
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
from unsloth import FastLanguageModel, FastModel
|
||||
from transformers import CsmForConditionalGeneration
|
||||
import torch
|
||||
# ruff: noqa
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from peft import PeftModel
|
||||
import warnings
|
||||
import requests
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.os_utils import require_package, require_python_package
|
||||
|
||||
require_package("ffmpeg", "ffmpeg")
|
||||
require_python_package("soundfile")
|
||||
require_python_package("snac")
|
||||
|
||||
import soundfile as sf
|
||||
from snac import SNAC
|
||||
snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz")
|
||||
snac_model = snac_model.to("cuda")
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 1: Loading Model and LoRA Adapters")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
model, tokenizer = FastLanguageModel.from_pretrained(
|
||||
model_name = "unsloth/orpheus-3b-0.1-ft",
|
||||
max_seq_length= 2048, # Choose any for long context!
|
||||
dtype = None, # Select None for auto detection
|
||||
load_in_4bit = False, # Select True for 4bit which reduces memory usage
|
||||
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
|
||||
)
|
||||
|
||||
base_model_class = model.__class__.__name__
|
||||
|
||||
|
||||
model = FastLanguageModel.get_peft_model(
|
||||
model,
|
||||
r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
|
||||
target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
|
||||
"gate_proj", "up_proj", "down_proj",],
|
||||
lora_alpha = 64,
|
||||
lora_dropout = 0, # Supports any, but = 0 is optimized
|
||||
bias = "none", # Supports any, but = "none" is optimized
|
||||
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
|
||||
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
|
||||
random_state = 3407,
|
||||
use_rslora = False, # We support rank stabilized LoRA
|
||||
loftq_config = None, # And LoftQ
|
||||
)
|
||||
print("✅ Model and LoRA adapters loaded successfully!")
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 2: Checking Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
assert isinstance(model, PeftModel), "Model should be an instance of PeftModel"
|
||||
print("✅ Model is an instance of PeftModel!")
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 3: Checking Config Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
def find_lora_base_model(model_to_inspect):
|
||||
current = model_to_inspect
|
||||
if hasattr(current, "base_model"):
|
||||
current = current.base_model
|
||||
if hasattr(current, "model"):
|
||||
current = current.model
|
||||
return current
|
||||
pass
|
||||
|
||||
|
||||
config_model = find_lora_base_model(model) if isinstance(model, PeftModel) else model
|
||||
|
||||
assert config_model.__class__.__name__ == base_model_class, f"Expected config_model class to be {base_model_class}"
|
||||
print("✅ config_model returns correct Base Model class:", str(base_model_class))
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 4: Saving and Merging Model")
|
||||
print(f"{'='*80}")
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error") # Treat warnings as errors
|
||||
try:
|
||||
model.save_pretrained_merged("orpheus", tokenizer)
|
||||
print("✅ Model saved and merged successfully without warnings!")
|
||||
except Exception as e:
|
||||
assert False, f"Model saving/merging failed with exception: {e}"
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 5: Loading Model for Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
model, tokenizer = FastLanguageModel.from_pretrained(
|
||||
model_name = "unsloth/orpheus-3b-0.1-ft",
|
||||
max_seq_length= 2048, # Choose any for long context!
|
||||
dtype = None, # Select None for auto detection
|
||||
load_in_4bit = False, # Select True for 4bit which reduces memory usage
|
||||
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
|
||||
)
|
||||
|
||||
#from transformers import AutoProcessor
|
||||
#processor = AutoProcessor.from_pretrained("unsloth/csm-1b")
|
||||
|
||||
print("✅ Model loaded for inference successfully!")
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 6: Running Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
#@title Run Inference
|
||||
|
||||
|
||||
FastLanguageModel.for_inference(model) # Enable native 2x faster inference
|
||||
|
||||
# Moving snac_model cuda to cpu
|
||||
snac_model.to("cpu")
|
||||
prompts = [
|
||||
"Hey there my name is Elise, <giggles> and I'm a speech generation model that can sound like a person.",
|
||||
]
|
||||
|
||||
chosen_voice = None # None for single-speaker
|
||||
|
||||
prompts_ = [(f"{chosen_voice}: " + p) if chosen_voice else p for p in prompts]
|
||||
|
||||
all_input_ids = []
|
||||
|
||||
for prompt in prompts_:
|
||||
input_ids = tokenizer(prompt, return_tensors="pt").input_ids
|
||||
all_input_ids.append(input_ids)
|
||||
|
||||
start_token = torch.tensor([[ 128259]], dtype=torch.int64) # Start of human
|
||||
end_tokens = torch.tensor([[128009, 128260]], dtype=torch.int64) # End of text, End of human
|
||||
|
||||
all_modified_input_ids = []
|
||||
for input_ids in all_input_ids:
|
||||
modified_input_ids = torch.cat([start_token, input_ids, end_tokens], dim=1) # SOH SOT Text EOT EOH
|
||||
all_modified_input_ids.append(modified_input_ids)
|
||||
|
||||
all_padded_tensors = []
|
||||
all_attention_masks = []
|
||||
max_length = max([modified_input_ids.shape[1] for modified_input_ids in all_modified_input_ids])
|
||||
for modified_input_ids in all_modified_input_ids:
|
||||
padding = max_length - modified_input_ids.shape[1]
|
||||
padded_tensor = torch.cat([torch.full((1, padding), 128263, dtype=torch.int64), modified_input_ids], dim=1)
|
||||
attention_mask = torch.cat([torch.zeros((1, padding), dtype=torch.int64), torch.ones((1, modified_input_ids.shape[1]), dtype=torch.int64)], dim=1)
|
||||
all_padded_tensors.append(padded_tensor)
|
||||
all_attention_masks.append(attention_mask)
|
||||
|
||||
all_padded_tensors = torch.cat(all_padded_tensors, dim=0)
|
||||
all_attention_masks = torch.cat(all_attention_masks, dim=0)
|
||||
|
||||
input_ids = all_padded_tensors.to("cuda")
|
||||
attention_mask = all_attention_masks.to("cuda")
|
||||
generated_ids = model.generate(
|
||||
input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
max_new_tokens=1200,
|
||||
do_sample=True,
|
||||
temperature=0.6,
|
||||
top_p=0.95,
|
||||
repetition_penalty=1.1,
|
||||
num_return_sequences=1,
|
||||
eos_token_id=128258,
|
||||
use_cache = True
|
||||
)
|
||||
token_to_find = 128257
|
||||
token_to_remove = 128258
|
||||
|
||||
token_indices = (generated_ids == token_to_find).nonzero(as_tuple=True)
|
||||
|
||||
if len(token_indices[1]) > 0:
|
||||
last_occurrence_idx = token_indices[1][-1].item()
|
||||
cropped_tensor = generated_ids[:, last_occurrence_idx+1:]
|
||||
else:
|
||||
cropped_tensor = generated_ids
|
||||
|
||||
mask = cropped_tensor != token_to_remove
|
||||
|
||||
processed_rows = []
|
||||
|
||||
for row in cropped_tensor:
|
||||
masked_row = row[row != token_to_remove]
|
||||
processed_rows.append(masked_row)
|
||||
|
||||
code_lists = []
|
||||
|
||||
for row in processed_rows:
|
||||
row_length = row.size(0)
|
||||
new_length = (row_length // 7) * 7
|
||||
trimmed_row = row[:new_length]
|
||||
trimmed_row = [t - 128266 for t in trimmed_row]
|
||||
code_lists.append(trimmed_row)
|
||||
|
||||
|
||||
def redistribute_codes(code_list):
|
||||
layer_1 = []
|
||||
layer_2 = []
|
||||
layer_3 = []
|
||||
for i in range((len(code_list)+1)//7):
|
||||
layer_1.append(code_list[7*i])
|
||||
layer_2.append(code_list[7*i+1]-4096)
|
||||
layer_3.append(code_list[7*i+2]-(2*4096))
|
||||
layer_3.append(code_list[7*i+3]-(3*4096))
|
||||
layer_2.append(code_list[7*i+4]-(4*4096))
|
||||
layer_3.append(code_list[7*i+5]-(5*4096))
|
||||
layer_3.append(code_list[7*i+6]-(6*4096))
|
||||
codes = [torch.tensor(layer_1).unsqueeze(0),
|
||||
torch.tensor(layer_2).unsqueeze(0),
|
||||
torch.tensor(layer_3).unsqueeze(0)]
|
||||
|
||||
# codes = [c.to("cuda") for c in codes]
|
||||
audio_hat = snac_model.decode(codes)
|
||||
return audio_hat
|
||||
|
||||
my_samples = []
|
||||
for code_list in code_lists:
|
||||
samples = redistribute_codes(code_list)
|
||||
my_samples.append(samples)
|
||||
output_path = "orpheus_audio.wav"
|
||||
try:
|
||||
for i, samples in enumerate(my_samples):
|
||||
audio_data = samples.detach().squeeze().cpu().numpy()
|
||||
import soundfile as sf
|
||||
sf.write(output_path, audio_data, 24000) # Explicitly pass sample rate
|
||||
print(f"✅ Audio saved to {output_path}!")
|
||||
except Exception as e:
|
||||
assert False, f"Inference failed with exception: {e}"
|
||||
|
||||
# Verify the file exists
|
||||
import os
|
||||
assert os.path.exists(output_path), f"Audio file not found at {output_path}"
|
||||
print("✅ Audio file exists on disk!")
|
||||
del my_samples, samples
|
||||
## assert that transcribed_text contains The birch canoe slid on the smooth planks. Glued the sheet to the dark blue background. It's easy to tell the depth of a well. Four hours of steady work faced us.
|
||||
|
||||
print("✅ All sections passed successfully!")
|
||||
|
||||
|
||||
safe_remove_directory("./unsloth_compiled_cache")
|
||||
safe_remove_directory("./orpheus")
|
||||
189
tests/saving/text_to_speech_models/test_whisper.py
Normal file
189
tests/saving/text_to_speech_models/test_whisper.py
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
from unsloth import FastLanguageModel, FastModel
|
||||
from transformers import WhisperForConditionalGeneration, WhisperProcessor
|
||||
import torch
|
||||
# ruff: noqa
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from peft import PeftModel
|
||||
import warnings
|
||||
import requests
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.os_utils import require_package, require_python_package
|
||||
|
||||
require_package("ffmpeg", "ffmpeg")
|
||||
require_python_package("soundfile")
|
||||
|
||||
import soundfile as sf
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 1: Loading Model and LoRA Adapters")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
model, tokenizer = FastModel.from_pretrained(
|
||||
model_name = "unsloth/whisper-large-v3",
|
||||
dtype = None, # Leave as None for auto detection
|
||||
load_in_4bit = False, # Set to True to do 4bit quantization which reduces memory
|
||||
auto_model = WhisperForConditionalGeneration,
|
||||
whisper_language = "English",
|
||||
whisper_task = "transcribe",
|
||||
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
|
||||
)
|
||||
|
||||
|
||||
base_model_class = model.__class__.__name__
|
||||
#https://github.com/huggingface/transformers/issues/37172
|
||||
model.generation_config.input_ids = model.generation_config.forced_decoder_ids
|
||||
model.generation_config.forced_decoder_ids = None
|
||||
|
||||
|
||||
model = FastModel.get_peft_model(
|
||||
model,
|
||||
r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
|
||||
target_modules = ["q_proj", "v_proj"],
|
||||
lora_alpha = 64,
|
||||
lora_dropout = 0, # Supports any, but = 0 is optimized
|
||||
bias = "none", # Supports any, but = "none" is optimized
|
||||
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
|
||||
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
|
||||
random_state = 3407,
|
||||
use_rslora = False, # We support rank stabilized LoRA
|
||||
loftq_config = None, # And LoftQ
|
||||
task_type = None, # ** MUST set this for Whisper **
|
||||
)
|
||||
|
||||
print("✅ Model and LoRA adapters loaded successfully!")
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 2: Checking Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
assert isinstance(model, PeftModel), "Model should be an instance of PeftModel"
|
||||
print("✅ Model is an instance of PeftModel!")
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 3: Checking Config Model Class Type")
|
||||
print(f"{'='*80}")
|
||||
|
||||
def find_lora_base_model(model_to_inspect):
|
||||
current = model_to_inspect
|
||||
if hasattr(current, "base_model"):
|
||||
current = current.base_model
|
||||
if hasattr(current, "model"):
|
||||
current = current.model
|
||||
return current
|
||||
pass
|
||||
|
||||
|
||||
config_model = find_lora_base_model(model) if isinstance(model, PeftModel) else model
|
||||
|
||||
assert config_model.__class__.__name__ == base_model_class, f"Expected config_model class to be {base_model_class}"
|
||||
print("✅ config_model returns correct Base Model class:", str(base_model_class))
|
||||
|
||||
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 4: Saving and Merging Model")
|
||||
print(f"{'='*80}")
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error") # Treat warnings as errors
|
||||
try:
|
||||
model.save_pretrained_merged("whisper", tokenizer)
|
||||
print("✅ Model saved and merged successfully without warnings!")
|
||||
except Exception as e:
|
||||
assert False, f"Model saving/merging failed with exception: {e}"
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 5: Loading Model for Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
model, tokenizer = FastModel.from_pretrained(
|
||||
model_name = "./whisper",
|
||||
dtype = None, # Leave as None for auto detection
|
||||
load_in_4bit = False, # Set to True to do 4bit quantization which reduces memory
|
||||
auto_model = WhisperForConditionalGeneration,
|
||||
whisper_language = "English",
|
||||
whisper_task = "transcribe",
|
||||
# token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
|
||||
)
|
||||
|
||||
# model = WhisperForConditionalGeneration.from_pretrained("./whisper")
|
||||
# processor = WhisperProcessor.from_pretrained("./whisper")
|
||||
|
||||
print("✅ Model loaded for inference successfully!")
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 6: Downloading Sample Audio File")
|
||||
print(f"{'='*80}")
|
||||
|
||||
audio_url = "https://upload.wikimedia.org/wikipedia/commons/5/5b/Speech_12dB_s16.flac"
|
||||
audio_file = "Speech_12dB_s16.flac"
|
||||
|
||||
try:
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
|
||||
}
|
||||
response = requests.get(audio_url, headers=headers)
|
||||
response.raise_for_status()
|
||||
with open(audio_file, "wb") as f:
|
||||
f.write(response.content)
|
||||
print("✅ Audio file downloaded successfully!")
|
||||
except Exception as e:
|
||||
assert False, f"Failed to download audio file: {e}"
|
||||
|
||||
print(f"\n{'='*80}")
|
||||
print("🔍 SECTION 7: Running Inference")
|
||||
print(f"{'='*80}")
|
||||
|
||||
|
||||
from transformers import pipeline
|
||||
import torch
|
||||
FastModel.for_inference(model)
|
||||
model.eval()
|
||||
#Create pipeline without specifying the device
|
||||
whisper = pipeline(
|
||||
"automatic-speech-recognition",
|
||||
model=model,
|
||||
tokenizer=tokenizer.tokenizer,
|
||||
feature_extractor=tokenizer.feature_extractor,
|
||||
processor=tokenizer,
|
||||
return_language=True,
|
||||
torch_dtype=torch.float16 # Remove the device parameter
|
||||
)
|
||||
# Example usage
|
||||
audio_file = "Speech_12dB_s16.flac"
|
||||
transcribed_text = whisper(audio_file)
|
||||
# audio, sr = sf.read(audio_file)
|
||||
# input_features = processor(audio, return_tensors="pt").input_features
|
||||
# transcribed_text = model.generate(input_features=input_features)
|
||||
print(f"📝 Transcribed Text: {transcribed_text['text']}")
|
||||
|
||||
## assert that transcribed_text contains The birch canoe slid on the smooth planks. Glued the sheet to the dark blue background. It's easy to tell the depth of a well. Four hours of steady work faced us.
|
||||
|
||||
expected_phrases = [
|
||||
"birch canoe slid on the smooth planks",
|
||||
"sheet to the dark blue background",
|
||||
"easy to tell the depth of a well",
|
||||
"Four hours of steady work faced us",
|
||||
]
|
||||
|
||||
transcribed_lower = transcribed_text["text"].lower()
|
||||
all_phrases_found = all(phrase.lower() in transcribed_lower for phrase in expected_phrases)
|
||||
|
||||
assert all_phrases_found, f"Expected phrases not found in transcription: {transcribed_text['text']}"
|
||||
print("✅ Transcription contains all expected phrases!")
|
||||
|
||||
|
||||
safe_remove_directory("./unsloth_compiled_cache")
|
||||
safe_remove_directory("./whisper")
|
||||
|
|
@ -11,8 +11,9 @@ from huggingface_hub import HfFileSystem
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
|
||||
|
|
|
|||
|
|
@ -11,8 +11,10 @@ from trl import SFTTrainer, SFTConfig
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
|
||||
|
|
|
|||
|
|
@ -11,8 +11,9 @@ from trl import SFTTrainer, SFTConfig
|
|||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parents[3]
|
||||
sys.path.append(str(REPO_ROOT))
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tests.utils.cleanup_utils import safe_remove_directory
|
||||
from tests.utils.ocr_eval import OCRModelEvaluator
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ for config_name, params in configs.items():
|
|||
evaluator.print_model_comparison()
|
||||
```
|
||||
|
||||
### Handwritting Recognition
|
||||
### Handwriting Recognition
|
||||
```python
|
||||
# Specialized evaluation for handwriting
|
||||
def evaluate_handwriting_models(models, handwriting_dataset):
|
||||
|
|
|
|||
119
tests/utils/os_utils.py
Normal file
119
tests/utils/os_utils.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import subprocess
|
||||
import sys
|
||||
import os
|
||||
import shutil
|
||||
import importlib
|
||||
|
||||
def detect_package_manager():
|
||||
"""Detect the available package manager"""
|
||||
package_managers = {
|
||||
'apt': '/usr/bin/apt',
|
||||
'yum': '/usr/bin/yum',
|
||||
'dnf': '/usr/bin/dnf',
|
||||
'pacman': '/usr/bin/pacman',
|
||||
'zypper': '/usr/bin/zypper'
|
||||
}
|
||||
|
||||
for pm, path in package_managers.items():
|
||||
if os.path.exists(path):
|
||||
return pm
|
||||
return None
|
||||
|
||||
def check_package_installed(package_name, package_manager=None):
|
||||
"""Check if a package is installed using the system package manager"""
|
||||
|
||||
if package_manager is None:
|
||||
package_manager = detect_package_manager()
|
||||
|
||||
if package_manager is None:
|
||||
print("Warning: Could not detect package manager")
|
||||
return None
|
||||
|
||||
try:
|
||||
if package_manager == 'apt':
|
||||
# Check with dpkg
|
||||
result = subprocess.run(['dpkg', '-l', package_name],
|
||||
capture_output=True, text=True)
|
||||
return result.returncode == 0
|
||||
|
||||
elif package_manager in ['yum', 'dnf']:
|
||||
# Check with rpm
|
||||
result = subprocess.run(['rpm', '-q', package_name],
|
||||
capture_output=True, text=True)
|
||||
return result.returncode == 0
|
||||
|
||||
elif package_manager == 'pacman':
|
||||
result = subprocess.run(['pacman', '-Q', package_name],
|
||||
capture_output=True, text=True)
|
||||
return result.returncode == 0
|
||||
|
||||
elif package_manager == 'zypper':
|
||||
result = subprocess.run(['zypper', 'se', '-i', package_name],
|
||||
capture_output=True, text=True)
|
||||
return package_name in result.stdout
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error checking package: {e}")
|
||||
return None
|
||||
|
||||
def require_package(package_name, executable_name=None):
|
||||
"""Require a package to be installed, exit if not found"""
|
||||
|
||||
# First check if executable is in PATH (most reliable)
|
||||
if executable_name:
|
||||
if shutil.which(executable_name):
|
||||
print(f"✓ {executable_name} is available")
|
||||
return
|
||||
|
||||
# Then check with package manager
|
||||
pm = detect_package_manager()
|
||||
is_installed = check_package_installed(package_name, pm)
|
||||
|
||||
if is_installed:
|
||||
print(f"✓ Package {package_name} is installed")
|
||||
return
|
||||
|
||||
# Package not found - show installation instructions
|
||||
print(f"❌ Error: {package_name} is not installed")
|
||||
print(f"\nPlease install {package_name} using your system package manager:")
|
||||
|
||||
install_commands = {
|
||||
'apt': f"sudo apt update && sudo apt install {package_name}",
|
||||
'yum': f"sudo yum install {package_name}",
|
||||
'dnf': f"sudo dnf install {package_name}",
|
||||
'pacman': f"sudo pacman -S {package_name}",
|
||||
'zypper': f"sudo zypper install {package_name}"
|
||||
}
|
||||
|
||||
if pm and pm in install_commands:
|
||||
print(f" {install_commands[pm]}")
|
||||
else:
|
||||
for pm_name, cmd in install_commands.items():
|
||||
print(f" {pm_name}: {cmd}")
|
||||
|
||||
print(f"\nAlternatively, install with conda:")
|
||||
print(f" conda install -c conda-forge {package_name}")
|
||||
|
||||
print(f"\nPlease install the required package and run the script again.")
|
||||
sys.exit(1)
|
||||
|
||||
# Usage
|
||||
#require_package("ffmpeg", "ffmpeg")
|
||||
|
||||
def require_python_package(package_name, import_name=None, pip_name=None):
|
||||
"""Require a Python package to be installed, exit if not found"""
|
||||
if import_name is None:
|
||||
import_name = package_name
|
||||
if pip_name is None:
|
||||
pip_name = package_name
|
||||
|
||||
if importlib.util.find_spec(import_name) is None:
|
||||
print(f"❌ Error: Python package '{package_name}' is not installed")
|
||||
print(f"\nPlease install {package_name} using pip:")
|
||||
print(f" pip install {pip_name}")
|
||||
print(f" # or with conda:")
|
||||
print(f" conda install {pip_name}")
|
||||
print(f"\nAfter installation, run this script again.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"✓ Python package '{package_name}' is installed")
|
||||
|
|
@ -60,9 +60,13 @@ keynames = "\n" + "\n".join(os.environ.keys())
|
|||
if "HF_XET_HIGH_PERFORMANCE" not in os.environ:
|
||||
os.environ["HF_XET_HIGH_PERFORMANCE"] = "1"
|
||||
pass
|
||||
if "\nCOLAB_" in keynames:
|
||||
# Disable XET cache sine it eats too much space
|
||||
if "HF_XET_CHUNK_CACHE_SIZE_BYTES" not in os.environ:
|
||||
os.environ["HF_XET_CHUNK_CACHE_SIZE_BYTES"] = "0"
|
||||
pass
|
||||
if "\nCOLAB_" in keynames:
|
||||
os.environ["HF_XET_RECONSTRUCT_WRITE_SEQUENTIALLY"] = "0"
|
||||
pass
|
||||
|
||||
# Log Unsloth is being used
|
||||
os.environ["UNSLOTH_IS_PRESENT"] = "1"
|
||||
|
|
@ -89,7 +93,7 @@ DEVICE_TYPE : str = get_device_type()
|
|||
|
||||
# Reduce VRAM usage by reducing fragmentation
|
||||
# And optimize pinning of memory
|
||||
if DEVICE_TYPE == "cuda":
|
||||
if DEVICE_TYPE == "cuda" and os.environ.get("UNSLOTH_VLLM_STANDBY", "0")=="0":
|
||||
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = \
|
||||
"expandable_segments:True,"\
|
||||
"roundup_power2_divisions:[32:256,64:128,256:64,>:32]"
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ dX backward kernel
|
|||
`fused_mul` notes:
|
||||
- In the forward pass, if we used the multiplication of topk weights (e.g., in the second grouped GEMM in fused MoE MLP), we need to make a few additional changes:
|
||||
1) We load topk_weights in natural (token) order. Since we only enable `fuse_mul` when permuting on store (`permute_y`), we multiply grad_output by topk_weights before backpropagating
|
||||
2) We need to calculate the gradient of the topk_weights. This gets messy since we need do an additioanl elementwise multiplication in the GEMM main loop and then write out in unpermuted order. For now, we do not fuse this step but calculate as a simple
|
||||
2) We need to calculate the gradient of the topk_weights. This gets messy since we need do an additional elementwise multiplication in the GEMM main loop and then write out in unpermuted order. For now, we do not fuse this step but calculate as a simple
|
||||
|
||||
Invalid combinations:
|
||||
- permute_y and use_tma_load: permuting y on store in forward -> load in permuted order in backward, therefore can't use TMA load (unless Blackwell which supports gather / scatter TMA)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
__version__ = "2025.6.1"
|
||||
__version__ = "2025.6.5"
|
||||
|
||||
__all__ = [
|
||||
"SUPPORTS_BFLOAT16",
|
||||
|
|
@ -755,24 +755,10 @@ exec(BitsAndBytesConfig__init__, globals())
|
|||
|
||||
if torch.cuda.device_count() == 1:
|
||||
from accelerate.utils.dataclasses import DistributedType
|
||||
def _prepare_backend(
|
||||
self, cpu = False, sagemaker_dp = False, backend: str = None,
|
||||
) -> tuple[str, DistributedType]:
|
||||
return None, DistributedType.NO
|
||||
pass
|
||||
def _prepare_backend(self, *args, **kwargs): return None, DistributedType.NO
|
||||
import accelerate.state
|
||||
accelerate.state.PartialState._prepare_backend = _prepare_backend
|
||||
|
||||
import accelerate.accelerator
|
||||
prepare = inspect.getsource(accelerate.accelerator.Accelerator.prepare)
|
||||
prepare = prepare.split("\n")
|
||||
spaces = prepare[0].find("def")
|
||||
prepare = "\n".join(x[spaces:] for x in prepare)
|
||||
x = "for obj in args:"
|
||||
s = " "*spaces
|
||||
prepare = prepare.replace(x, f'self.state.distributed_type = DistributedType.NO\n{s}{x}', 1)
|
||||
exec(prepare, globals())
|
||||
accelerate.accelerator.Accelerator.prepare = prepare
|
||||
accelerate.accelerator.Accelerator.distributed_type = lambda *args, **kwargs: DistributedType.NO
|
||||
pass
|
||||
|
||||
import transformers.utils.quantization_config
|
||||
|
|
|
|||
|
|
@ -466,7 +466,7 @@ class FastCohereModel(FastLlamaModel):
|
|||
CohereDecoderLayer .forward = CohereDecoderLayer_fast_forward
|
||||
CohereModel .forward = LlamaModel_fast_forward
|
||||
CohereForCausalLM .forward = CausalLM_fast_forward(CohereModel_fast_forward_inference)
|
||||
PeftModelForCausalLM .forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM .forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(CohereForCausalLM)
|
||||
|
||||
import transformers.models.cohere.modeling_cohere
|
||||
|
|
|
|||
|
|
@ -332,7 +332,7 @@ class FastGemmaModel(FastLlamaModel):
|
|||
GemmaDecoderLayer .forward = GemmaDecoderLayer_fast_forward
|
||||
GemmaModel .forward = LlamaModel_fast_forward
|
||||
GemmaForCausalLM .forward = CausalLM_fast_forward(GemmaModel_fast_forward_inference)
|
||||
PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM.forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(GemmaForCausalLM)
|
||||
|
||||
# Solves https://github.com/unslothai/unsloth/issues/168
|
||||
|
|
|
|||
|
|
@ -477,7 +477,7 @@ class FastGemma2Model(FastLlamaModel):
|
|||
Gemma2DecoderLayer .forward = Gemma2DecoderLayer_fast_forward
|
||||
Gemma2Model .forward = LlamaModel_fast_forward
|
||||
Gemma2ForCausalLM .forward = CausalLM_fast_forward(Gemma2Model_fast_forward_inference)
|
||||
PeftModelForCausalLM .forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM .forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(Gemma2ForCausalLM)
|
||||
|
||||
# Solves https://github.com/unslothai/unsloth/issues/168
|
||||
|
|
|
|||
|
|
@ -468,7 +468,7 @@ class FastGraniteModel(FastLlamaModel):
|
|||
GraniteModel .forward = LlamaModel_fast_forward
|
||||
GraniteForCausalLM .forward = CausalLM_fast_forward(GraniteModel_fast_forward_inference)
|
||||
GraniteForCausalLM .__init__ = patched_init(GraniteForCausalLM.__init__)
|
||||
PeftModelForCausalLM .forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM .forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(GraniteForCausalLM)
|
||||
|
||||
import transformers.models.granite.modeling_granite
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ except:
|
|||
LlamaFlashAttention2 = LlamaAttention
|
||||
pass
|
||||
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, AutoConfig
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification, BitsAndBytesConfig, AutoConfig
|
||||
from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING
|
||||
from transformers import set_seed as transformers_set_seed
|
||||
from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model
|
||||
|
|
@ -1242,7 +1242,7 @@ pass
|
|||
|
||||
|
||||
@torch._disable_dynamo
|
||||
def PeftModelForCausalLM_fast_forward(
|
||||
def PeftModel_fast_forward(
|
||||
self,
|
||||
input_ids = None,
|
||||
causal_mask = None,
|
||||
|
|
@ -1257,19 +1257,33 @@ def PeftModelForCausalLM_fast_forward(
|
|||
logits_to_keep = 0,
|
||||
**kwargs,
|
||||
):
|
||||
return self.base_model(
|
||||
input_ids = input_ids,
|
||||
causal_mask = causal_mask,
|
||||
attention_mask = attention_mask,
|
||||
inputs_embeds = inputs_embeds,
|
||||
labels = labels,
|
||||
output_attentions = output_attentions,
|
||||
output_hidden_states = output_hidden_states,
|
||||
return_dict = return_dict,
|
||||
num_logits_to_keep = num_logits_to_keep,
|
||||
logits_to_keep = logits_to_keep,
|
||||
**kwargs,
|
||||
)
|
||||
is_classification = "Classification" in str(type( self.base_model.model))
|
||||
if is_classification:
|
||||
#causal_mask = causal_mask,
|
||||
return self.base_model(
|
||||
input_ids = input_ids,
|
||||
attention_mask = attention_mask,
|
||||
inputs_embeds = inputs_embeds,
|
||||
labels = labels,
|
||||
output_attentions = output_attentions,
|
||||
output_hidden_states = output_hidden_states,
|
||||
return_dict = return_dict,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
return self.base_model(
|
||||
input_ids = input_ids,
|
||||
causal_mask = causal_mask,
|
||||
attention_mask = attention_mask,
|
||||
inputs_embeds = inputs_embeds,
|
||||
labels = labels,
|
||||
output_attentions = output_attentions,
|
||||
output_hidden_states = output_hidden_states,
|
||||
return_dict = return_dict,
|
||||
num_logits_to_keep = num_logits_to_keep,
|
||||
logits_to_keep = logits_to_keep,
|
||||
**kwargs,
|
||||
)
|
||||
pass
|
||||
|
||||
|
||||
|
|
@ -1667,7 +1681,7 @@ class FastLlamaModel:
|
|||
LlamaDecoderLayer .forward = LlamaDecoderLayer_fast_forward
|
||||
LlamaModel .forward = LlamaModel_fast_forward
|
||||
LlamaForCausalLM .forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference)
|
||||
PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM.forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(LlamaForCausalLM)
|
||||
|
||||
# Solves https://github.com/unslothai/unsloth/issues/168
|
||||
|
|
@ -1684,17 +1698,18 @@ class FastLlamaModel:
|
|||
|
||||
@staticmethod
|
||||
def from_pretrained(
|
||||
model_name = "unsloth/llama-3-8b-bnb-4bit",
|
||||
max_seq_length = None,
|
||||
dtype = None,
|
||||
load_in_4bit = True,
|
||||
token = None,
|
||||
device_map = "sequential",
|
||||
rope_scaling = None,
|
||||
fix_tokenizer = True,
|
||||
model_patcher = None,
|
||||
tokenizer_name = None,
|
||||
trust_remote_code = False,
|
||||
model_name = "unsloth/llama-3-8b-bnb-4bit",
|
||||
max_seq_length = None,
|
||||
dtype = None,
|
||||
load_in_4bit = True,
|
||||
token = None,
|
||||
device_map = "sequential",
|
||||
rope_scaling = None,
|
||||
fix_tokenizer = True,
|
||||
model_patcher = None,
|
||||
tokenizer_name = None,
|
||||
trust_remote_code = False,
|
||||
revision = None,
|
||||
|
||||
fast_inference = False, # uses vLLM
|
||||
gpu_memory_utilization = 0.5,
|
||||
|
|
@ -1702,6 +1717,8 @@ class FastLlamaModel:
|
|||
random_state = 3407,
|
||||
max_lora_rank = 16,
|
||||
disable_log_stats = False,
|
||||
unsloth_vllm_standby = False,
|
||||
num_labels = None,
|
||||
**kwargs,
|
||||
):
|
||||
os.environ["UNSLOTH_USE_NEW_MODEL"] = "0"
|
||||
|
|
@ -1721,6 +1738,8 @@ class FastLlamaModel:
|
|||
if major_version < 7:
|
||||
print("Unsloth: vLLM does not work on older GPUs - will switch to Unsloth inference!")
|
||||
fast_inference = False
|
||||
if unsloth_vllm_standby and os.environ.get("UNSLOTH_VLLM_STANDBY", "0") == "0":
|
||||
raise RuntimeError("Unsloth: `unsloth_vllm_standby` is True, but environment variable `UNSLOTH_VLLM_STANDBY` is not set to 1!")
|
||||
pass
|
||||
|
||||
if token is None: token = get_token()
|
||||
|
|
@ -1836,7 +1855,20 @@ class FastLlamaModel:
|
|||
# Cannot be None, since HF now checks for the config
|
||||
if load_in_4bit: kwargs["quantization_config"] = bnb_config
|
||||
|
||||
if not fast_inference:
|
||||
if num_labels is not None:
|
||||
model = AutoModelForSequenceClassification.from_pretrained(
|
||||
model_name,
|
||||
device_map = device_map,
|
||||
torch_dtype = dtype,
|
||||
num_labels = num_labels,
|
||||
#quantization_config = bnb_config,
|
||||
token = token,
|
||||
max_position_embeddings = max_position_embeddings,
|
||||
trust_remote_code = trust_remote_code,
|
||||
attn_implementation = "eager",
|
||||
**kwargs,
|
||||
)
|
||||
elif not fast_inference:
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name,
|
||||
device_map = device_map,
|
||||
|
|
@ -1869,6 +1901,7 @@ class FastLlamaModel:
|
|||
max_lora_rank = max_lora_rank,
|
||||
disable_log_stats = disable_log_stats,
|
||||
use_bitsandbytes = load_in_4bit,
|
||||
unsloth_vllm_standby = unsloth_vllm_standby,
|
||||
)
|
||||
for allowed_arg in allowed_args:
|
||||
if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs:
|
||||
|
|
@ -2391,7 +2424,12 @@ class FastLlamaModel:
|
|||
raise NotImplementedError("Unsloth: Currently fast inference does not work with using biases for LoRA.")
|
||||
pass
|
||||
|
||||
#does not get lora yet, so get name from model, not base model
|
||||
|
||||
is_classification = "Classification" in str(type(model))
|
||||
# Get LoRA
|
||||
# if not is_classification else TaskType.SEQ_CLS
|
||||
|
||||
arguments = dict(
|
||||
r = r,
|
||||
lora_alpha = lora_alpha,
|
||||
|
|
@ -2414,8 +2452,11 @@ class FastLlamaModel:
|
|||
lora_config = LoraConfig(**arguments)
|
||||
|
||||
# First offload lm_head and embed_tokens to disk
|
||||
input_embeddings_device = model. get_input_embeddings().weight.device
|
||||
output_embeddings_device = model.get_output_embeddings().weight.device
|
||||
input_embeddings_device = model.get_input_embeddings().weight.device
|
||||
if is_classification:
|
||||
output_embeddings_device = model.score.weight.device
|
||||
else:
|
||||
output_embeddings_device = model.get_output_embeddings().weight.device
|
||||
|
||||
if use_gradient_checkpointing == "unsloth":
|
||||
if train_embed_tokens:
|
||||
|
|
|
|||
|
|
@ -869,6 +869,16 @@ __INT_TO_FLOAT_MAPPER = \
|
|||
"deepseek-ai/DeepSeek-R1-0528-Qwen3-8B",
|
||||
"unsloth/DeepSeek-R1-0528-Qwen3-8B-bnb-4bit",
|
||||
),
|
||||
"unsloth/Magistral-Small-2506-unsloth-bnb-4bit" : (
|
||||
"unsloth/Magistral-Small-2506",
|
||||
"mistralai/Magistral-Small-2506",
|
||||
"unsloth/Magistral-Small-2506-bnb-4bit",
|
||||
),
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-unsloth-bnb-4bit" : (
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
"mistralai/Mistral-Small-3.2-24B-Instruct-2506",
|
||||
"unsloth/Mistral-Small-3.2-24B-Instruct-2506-bnb-4bit",
|
||||
),
|
||||
}
|
||||
|
||||
INT_TO_FLOAT_MAPPER = {}
|
||||
|
|
|
|||
|
|
@ -368,7 +368,7 @@ class FastMistralModel(FastLlamaModel):
|
|||
MistralDecoderLayer .forward = LlamaDecoderLayer_fast_forward
|
||||
MistralModel .forward = LlamaModel_fast_forward
|
||||
MistralForCausalLM .forward = MistralForCausalLM_fast_forward
|
||||
PeftModelForCausalLM .forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM .forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(MistralForCausalLM)
|
||||
|
||||
# Solves https://github.com/unslothai/unsloth/issues/168
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ class FastQwen2Model(FastLlamaModel):
|
|||
Qwen2DecoderLayer .forward = LlamaDecoderLayer_fast_forward
|
||||
Qwen2Model .forward = LlamaModel_fast_forward
|
||||
Qwen2ForCausalLM .forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference)
|
||||
PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM.forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(Qwen2ForCausalLM)
|
||||
|
||||
# Solves https://github.com/unslothai/unsloth/issues/168
|
||||
|
|
|
|||
|
|
@ -387,7 +387,7 @@ class FastQwen3Model(FastLlamaModel):
|
|||
Qwen3DecoderLayer .forward = LlamaDecoderLayer_fast_forward
|
||||
Qwen3Model .forward = LlamaModel_fast_forward
|
||||
Qwen3ForCausalLM .forward = CausalLM_fast_forward(_LlamaModel_fast_forward_inference(Qwen3Attention_fast_forward_inference))
|
||||
PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM.forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(Qwen3ForCausalLM)
|
||||
|
||||
# Solves https://github.com/unslothai/unsloth/issues/168
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ class FastQwen3MoeModel(FastQwen3Model):
|
|||
Qwen3MoeDecoderLayer .forward = Qwen3MoeDecoderLayer_fast_forward
|
||||
Qwen3MoeModel .forward = LlamaModel_fast_forward
|
||||
Qwen3MoeForCausalLM .forward = CausalLM_fast_forward(LlamaModel_fast_forward_inference)
|
||||
PeftModelForCausalLM.forward = PeftModelForCausalLM_fast_forward
|
||||
PeftModelForCausalLM.forward = PeftModel_fast_forward
|
||||
fix_prepare_inputs_for_generation(Qwen3MoeForCausalLM)
|
||||
|
||||
# Solves https://github.com/unslothai/unsloth/issues/168
|
||||
|
|
|
|||
|
|
@ -471,6 +471,7 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
"num_generations" : 8,
|
||||
"top_k" : None,
|
||||
"vllm_mode" : "colocate",
|
||||
"generation_kwargs" : {},
|
||||
}
|
||||
for k, v in replacements.items():
|
||||
x = f"{k}( = [^,\n]{{1,}})?,\n"
|
||||
|
|
@ -510,6 +511,8 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
|
||||
# Check for loss_type = dr_grpo and scale_rewards for GRPO
|
||||
if "loss_type" in call_args and "scale_rewards" in call_args:
|
||||
# See https://github.com/huggingface/trl/issues/3130#issuecomment-2746947835
|
||||
# DAPO uses per token loss so BNPO loss used
|
||||
check_dr_grpo = \
|
||||
"if loss_type.lower() == 'dr_grpo':\n"\
|
||||
" loss_type = 'dr_grpo'\n"\
|
||||
|
|
@ -519,13 +522,16 @@ def _patch_trl_rl_trainers(trainer_file = "grpo_trainer"):
|
|||
" if scale_rewards == None:\n"\
|
||||
" scale_rewards = True\n"\
|
||||
" elif scale_rewards == True:\n"\
|
||||
" print('The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to force False.')\n"\
|
||||
" print('Unsloth: The Dr GRPO paper recommends setting `scale_rewards` to False! Will override. Set it to `None` to force False.')\n"\
|
||||
" scale_rewards = False\n"\
|
||||
"elif loss_type.lower() == 'dapo':\n"\
|
||||
" print('The DAPO paper recommends `mask_truncated_completions = True`')\n"\
|
||||
" print('The DAPO paper recommends `epsilon_high = 0.28`')\n"\
|
||||
" print('Unsloth: The DAPO paper recommends `mask_truncated_completions = True`')\n"\
|
||||
" print('Unsloth: The DAPO paper recommends `epsilon_high = 0.28`')\n"\
|
||||
" print('Unsloth: The DAPO paper recommends setting `beta = 0.0` to remove the KL term')\n"\
|
||||
" mask_truncated_completions = True\n"\
|
||||
" epsilon_high = 0.28\n"\
|
||||
" beta = 0.0\n"\
|
||||
" loss_type = 'bnpo'"
|
||||
"\n"
|
||||
extra_args += check_dr_grpo
|
||||
pass
|
||||
|
|
@ -645,6 +651,18 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
|
|||
init = inspect.getsource(RLTrainer.__init__)
|
||||
old_init = init
|
||||
|
||||
# Remove brackets in comments since it interferes ie (...)
|
||||
comments = re.findall(r"\#[^\n]{1,}\n", init)
|
||||
bracketed_comments = [x for x in comments if "(" in x or ")" in x]
|
||||
# Replace with [...] instead
|
||||
for bracketed_comment in bracketed_comments:
|
||||
init = init.replace(
|
||||
bracketed_comment,
|
||||
bracketed_comment.replace("(", "[").replace(")", "]"),
|
||||
)
|
||||
pass
|
||||
|
||||
|
||||
# Remove peft_config
|
||||
init = init.replace("elif peft_config is None:", "elif False:")
|
||||
init = init.replace("elif peft_config is not None:", "elif False:")
|
||||
|
|
@ -734,7 +752,7 @@ def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, import
|
|||
|
||||
if trl_version >= "0.18":
|
||||
# Replace LLM init with already existing vLLM engine for colocate mode
|
||||
vllm_llm_init_pattern = r"self\.llm\s*=\s*LLM\([^)]*\)*\)"
|
||||
vllm_llm_init_pattern = r"self\.llm\s*=\s*LLM\(.*?\)*\)\s*?\n(?!,)"
|
||||
vllm_llm_replacement = "self.llm = model.vllm_engine\n"
|
||||
new_vllm_part = re.sub(
|
||||
vllm_llm_init_pattern,
|
||||
|
|
|
|||
|
|
@ -171,24 +171,64 @@ RL_FUNCTIONS["sft_trainer"].append(sft_trainer_compute_loss)
|
|||
def grpo_trainer__prepare_inputs(function_name, function):
|
||||
if function_name != "_prepare_inputs": return function
|
||||
|
||||
if "with torch.inference_mode()" not in function: return function
|
||||
import re
|
||||
# Try to find the function signature and insert after it
|
||||
# This matches the function signature and any decorators/comments, then finds the first non-empty line after the signature
|
||||
pattern = r"(def _prepare_inputs\s*\([^\)]*\)\s*(->\s*[^:]+)?\s*:\s*\n)"
|
||||
match = re.search(pattern, function)
|
||||
if match:
|
||||
sig_end = match.end(1)
|
||||
rest = function[sig_end:]
|
||||
rest = re.sub(r"^[ \t]*self\.llm\.wake_up\(\)\s*\n", "", rest)
|
||||
rest = re.sub(r"^[ \t]*torch\.cuda\.empty_cache\(\)\s*\n", "", rest)
|
||||
rest = re.sub(r"^[ \t]*free, total = torch.cuda.mem_get_info\(\)\s*\n", "", rest)
|
||||
rest = re.sub(r"^[ \t]*print\(f?\".*cuda.*\"\)\s*\n", "", rest)
|
||||
insert = (
|
||||
" if hasattr(self, 'llm'):\n"
|
||||
" if getattr(self.llm.llm_engine.vllm_config.model_config, 'enable_sleep_mode', False):\n"
|
||||
" self.llm.wake_up()\n"
|
||||
)
|
||||
function = function[:sig_end] + insert + rest
|
||||
else:
|
||||
pattern2 = r"(def _prepare_inputs\(.*?\):\n(?:[ ]+#[^\n]*\n)+)"
|
||||
match2 = re.search(pattern2, function, flags=re.DOTALL)
|
||||
if match2:
|
||||
header_and_comments = match2.group(1)
|
||||
rest = function[len(header_and_comments):]
|
||||
rest = re.sub(r"^[ \t]*self\.llm\.wake_up\(\)\s*\n", "", rest)
|
||||
rest = re.sub(r"^[ \t]*torch\.cuda\.empty_cache\(\)\s*\n", "", rest)
|
||||
rest = re.sub(r"^[ \t]*free, total = torch.cuda.mem_get_info\(\)\s*\n", "", rest)
|
||||
rest = re.sub(r"^[ \t]*print\(f?\".*cuda.*\"\)\s*\n", "", rest)
|
||||
insert = (
|
||||
" if (hasattr(self, 'llm'):\n"
|
||||
" if getattr(self.llm.llm_engine.vllm_config.model_config, 'enable_sleep_mode', False):\n"
|
||||
" self.llm.wake_up()\n"
|
||||
)
|
||||
function = header_and_comments + insert + rest
|
||||
|
||||
# Add mixed precision training
|
||||
function = function.replace(
|
||||
"with torch.inference_mode():",
|
||||
|
||||
"with torch.inference_mode(), "\
|
||||
"torch.amp.autocast(device_type = 'cuda', "\
|
||||
"dtype = ((torch.float16 if os.environ.get('ACCELERATE_MIXED_PRECISION', 'fp16') == 'fp16' else torch.bfloat16) "\
|
||||
"if not torch.is_autocast_enabled('cuda') else nullcontext())"\
|
||||
"if os.environ.get('UNSLOTH_FORCE_FLOAT32', '0') == '0' else torch.float16):",
|
||||
)
|
||||
|
||||
# Disable attaching a float32 conversion hook which upcasts logits to FP32
|
||||
function = function.replace(
|
||||
"self.accelerator.unwrap_model(self.model)",
|
||||
"self.accelerator.unwrap_model(self.model, keep_fp32_wrapper = False)",
|
||||
)
|
||||
sleep_and_cache = (
|
||||
"if hasattr(self, 'llm'):\n"
|
||||
" if getattr(self.llm.llm_engine.vllm_config.model_config, 'enable_sleep_mode', False):\n"
|
||||
" self.llm.sleep(os.environ.get('VLLM_SLEEP_MODE', 1))\n"
|
||||
" "
|
||||
)
|
||||
if re.search(r"\n\s*return ", function):
|
||||
function = re.sub(r"(\n\s*)return ", f"\\1{sleep_and_cache}return ", function, count=1)
|
||||
else:
|
||||
function = function.rstrip() + "\n " + sleep_and_cache
|
||||
return function
|
||||
pass
|
||||
RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__prepare_inputs)
|
||||
|
|
@ -211,7 +251,7 @@ def grpo_trainer__get_per_token_logps(function_name, function):
|
|||
if function_name != "_get_per_token_logps": return function
|
||||
|
||||
def _get_per_token_logps(self, model, input_ids, attention_mask, logits_to_keep, calc_logprob_flag = None):
|
||||
if os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0' and not calc_logprob_flag:
|
||||
if os.environ.get('UNSLOTH_USE_NEW_MODEL', '0') == '0' and not calc_logprob_flag:
|
||||
return None # Unsloth efficient GRPO
|
||||
# Otherwise, calculate normally:
|
||||
if not hasattr(self, '_autocast_dtype'):
|
||||
|
|
@ -273,7 +313,7 @@ def grpo_trainer_compute_loss(function_name, function):
|
|||
logits_to_keep = completion_ids.size(1) # we only need to compute the logits for the completion tokens
|
||||
_input_ids = input_ids
|
||||
_logits_to_keep = logits_to_keep
|
||||
|
||||
|
||||
per_token_logps = self._get_per_token_logps(model, input_ids, attention_mask, logits_to_keep)
|
||||
|
||||
# Compute the KL divergence between the model and the reference model
|
||||
|
|
@ -293,33 +333,55 @@ def grpo_trainer_compute_loss(function_name, function):
|
|||
# loss = ((per_token_loss * completion_mask).sum(dim=1) / completion_mask.sum(dim=1)).mean()
|
||||
if "old_per_token_logps" in inputs.keys():
|
||||
old_hidden_states = inputs["old_per_token_logps"]
|
||||
else:
|
||||
else:
|
||||
old_hidden_states = None
|
||||
input_ids = input_ids[:, -logits_to_keep:]
|
||||
if per_token_logps is not None:
|
||||
|
||||
ref_per_token_logps = ref_per_token_logps[:, :-1, :] # (B, L-1, V), exclude the last logit: it corresponds to the next token pred
|
||||
per_token_logps = per_token_logps[:, :-1, :] # (B, L-1, V), exclude the last logit: it corresponds to the next token pred
|
||||
|
||||
loss, completion_length, mean_kl = grpo_compute_loss_slow(
|
||||
ref_per_token_logps, per_token_logps, old_hidden_states, input_ids, completion_mask, self.beta, advantages,
|
||||
ref_per_token_logps,
|
||||
per_token_logps,
|
||||
old_hidden_states,
|
||||
input_ids,
|
||||
completion_mask,
|
||||
self.beta,
|
||||
advantages,
|
||||
loss_type = self.args.loss_type,
|
||||
epsilon_low = self.epsilon_low, epsilon_high = self.epsilon_high,
|
||||
epsilon_low = self.epsilon_low,
|
||||
epsilon_high = self.epsilon_high,
|
||||
max_completion_length = self.args.max_completion_length,
|
||||
delta = self.args.delta,
|
||||
)
|
||||
else:
|
||||
if hasattr(self.args, "loss_type"):
|
||||
loss, completion_length, mean_kl = grpo_accumulated_loss(
|
||||
self, _input_ids, logits_to_keep, completion_mask, advantages, old_hidden_states,
|
||||
self,
|
||||
_input_ids,
|
||||
logits_to_keep,
|
||||
completion_mask,
|
||||
advantages,
|
||||
old_hidden_states,
|
||||
n_chunks = self.args.unsloth_num_chunks,
|
||||
loss_type = self.args.loss_type,
|
||||
epsilon_low = self.epsilon_low, epsilon_high = self.epsilon_high,
|
||||
epsilon_low = self.epsilon_low,
|
||||
epsilon_high = self.epsilon_high,
|
||||
max_completion_length = self.args.max_completion_length,
|
||||
delta = self.args.delta,
|
||||
)
|
||||
else:
|
||||
# to ensure backwards compatibility with trl 0.15.2 and maybe even 0.17
|
||||
loss, completion_length, mean_kl = grpo_accumulated_loss(
|
||||
self, _input_ids, logits_to_keep, completion_mask, advantages, old_hidden_states,
|
||||
self,
|
||||
_input_ids,
|
||||
logits_to_keep,
|
||||
completion_mask,
|
||||
advantages,
|
||||
old_hidden_states,
|
||||
n_chunks = self.args.unsloth_num_chunks,
|
||||
)
|
||||
)
|
||||
|
||||
# Log the metrics
|
||||
# completion_length = self.accelerator.gather_for_metrics(completion_mask.sum(1)).float().mean().item()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue