Torch 2.4, Xformers>0.0.27, TRL>0.9, Python 3.12 + bug fixes (#902)
* Update pyproject.toml * Update tokenizer_utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * Update _utils.py * Update _utils.py * Update tokenizer_utils.py * Update tokenizer_utils.py * fix_tokenizer * Update tokenizer_utils.py * Update tokenizer_utils.py * Update save.py * Update save.py * Update save.py * Update save.py * Update save.py * Update loader.py * Update pyproject.toml * Update _utils.py * Update gemma2.py * Update gemma2.py * Update _utils.py * gemma 2 mask * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update _utils.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update _utils.py * Update llama.py * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Torch 2.4 Xformers 0.0.27post2 * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Gemma 2 fixes * Update gemma2.py * Update llama.py * Update llama.py * Update save.py * Update save.py
This commit is contained in:
Daniel Han
GitHub
parent
d8eb7b3a0c
commit
a08f36b754
6 changed files with 240 additions and 74 deletions
133
pyproject.toml
133
pyproject.toml
|
|
@ -43,111 +43,154 @@ huggingface = [
|
|||
"wheel>=0.42.0",
|
||||
"numpy",
|
||||
"accelerate>=0.26.1",
|
||||
"trl>=0.7.9,<0.9.0",
|
||||
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3",
|
||||
"peft>=0.7.1,!=0.11.0",
|
||||
"protobuf<4.0.0",
|
||||
"huggingface_hub",
|
||||
"hf_transfer",
|
||||
]
|
||||
cu118only = [
|
||||
"xformers==0.0.22.post7",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu121only = [
|
||||
"xformers==0.0.22.post7",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu118onlytorch211 = [
|
||||
"xformers==0.0.23",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu121onlytorch211 = [
|
||||
"xformers==0.0.23",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu118onlytorch212 = [
|
||||
"xformers==0.0.23.post1",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu121onlytorch212 = [
|
||||
"xformers==0.0.23.post1",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu118onlytorch220 = [
|
||||
"xformers==0.0.24",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu121onlytorch220 = [
|
||||
"xformers==0.0.24",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
]
|
||||
cu118onlytorch230 = [
|
||||
"xformers==0.0.26.post1",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'",
|
||||
]
|
||||
cu121onlytorch230 = [
|
||||
"xformers==0.0.26.post1",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'",
|
||||
]
|
||||
cu118onlytorch240 = [
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'",
|
||||
]
|
||||
cu121onlytorch240 = [
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'",
|
||||
"xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'",
|
||||
]
|
||||
|
||||
cu118 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118only]",
|
||||
]
|
||||
cu121 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121only]",
|
||||
]
|
||||
cu118-torch211 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch211]",
|
||||
]
|
||||
cu121-torch211 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch211]",
|
||||
]
|
||||
cu118-torch212 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch212]",
|
||||
]
|
||||
cu121-torch212 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch212]",
|
||||
]
|
||||
cu118-torch220 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch220]",
|
||||
]
|
||||
cu121-torch220 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch220]",
|
||||
]
|
||||
cu118-torch230 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch230]",
|
||||
]
|
||||
cu121-torch230 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch230]",
|
||||
]
|
||||
cu118-torch240 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch240]",
|
||||
]
|
||||
cu121-torch240 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch240]",
|
||||
]
|
||||
kaggle = [
|
||||
"unsloth[huggingface]",
|
||||
]
|
||||
kaggle-new = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
]
|
||||
conda = [
|
||||
"unsloth[huggingface]",
|
||||
]
|
||||
colab-torch211 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch211]",
|
||||
]
|
||||
colab-ampere-torch211 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch211]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -155,12 +198,12 @@ colab-ampere-torch211 = [
|
|||
]
|
||||
colab-torch220 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch220]",
|
||||
]
|
||||
colab-ampere-torch220 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch220]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -182,10 +225,10 @@ colab-new = [
|
|||
]
|
||||
colab-no-deps = [
|
||||
"accelerate>=0.26.1",
|
||||
"trl>=0.7.9",
|
||||
"trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3",
|
||||
"peft>=0.7.1",
|
||||
"xformers<0.0.27",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"protobuf<4.0.0",
|
||||
]
|
||||
colab = [
|
||||
|
|
@ -199,7 +242,7 @@ colab-ampere = [
|
|||
]
|
||||
cu118-ampere = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118only]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -207,7 +250,7 @@ cu118-ampere = [
|
|||
]
|
||||
cu121-ampere = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121only]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -215,7 +258,7 @@ cu121-ampere = [
|
|||
]
|
||||
cu118-ampere-torch211 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch211]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -223,7 +266,7 @@ cu118-ampere-torch211 = [
|
|||
]
|
||||
cu121-ampere-torch211 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch211]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -231,7 +274,7 @@ cu121-ampere-torch211 = [
|
|||
]
|
||||
cu118-ampere-torch220 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch220]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -239,7 +282,7 @@ cu118-ampere-torch220 = [
|
|||
]
|
||||
cu121-ampere-torch220 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch220]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -247,7 +290,7 @@ cu121-ampere-torch220 = [
|
|||
]
|
||||
cu118-ampere-torch230 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch230]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
|
|
@ -255,12 +298,28 @@ cu118-ampere-torch230 = [
|
|||
]
|
||||
cu121-ampere-torch230 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch230]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
"flash-attn>=2.6.3",
|
||||
]
|
||||
cu118-ampere-torch240 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu118onlytorch240]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
"flash-attn>=2.6.3",
|
||||
]
|
||||
cu121-ampere-torch240 = [
|
||||
"unsloth[huggingface]",
|
||||
"bitsandbytes>=0.43.3",
|
||||
"unsloth[cu121onlytorch240]",
|
||||
"packaging",
|
||||
"ninja",
|
||||
"flash-attn>=2.6.3",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
homepage = "http://www.unsloth.ai"
|
||||
|
|
|
|||
|
|
@ -53,7 +53,9 @@ from packaging.version import Version
|
|||
# Disable some warnings which can get annoying
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "huggingface_hub")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "trl")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "huggingface_hub")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "xformers")
|
||||
warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "subprocess")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "transformers")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "accelerate")
|
||||
|
|
@ -133,6 +135,28 @@ else:
|
|||
pass
|
||||
# =============================================
|
||||
|
||||
# =============================================
|
||||
# Fix KeyError: 'Cache only has 0 layers, attempted to access layer with index 0'
|
||||
import transformers.cache_utils
|
||||
if hasattr(transformers.cache_utils, "DynamicCache") and \
|
||||
transformers.cache_utils.DynamicCache.__getitem__.__name__ != "__cache_utils_getitem__":
|
||||
|
||||
source = inspect.getsource(transformers.cache_utils.DynamicCache.__getitem__)
|
||||
start = source.find("def")
|
||||
spaces = start*" "
|
||||
source = source.split("\n")
|
||||
source = "\n".join(x[start:] for x in source)
|
||||
where = source.find("raise KeyError")
|
||||
source = source[:where] + \
|
||||
f"if len(self) == 0:\n{spaces}{spaces}"\
|
||||
" raise RuntimeError('Unsloth: You must call `FastLanguageModel.for_inference(model)` before doing inference for Unsloth models.')\n" + \
|
||||
f"{spaces}{spaces}else:\n{spaces}{spaces}{spaces}" + source[where:]
|
||||
source = source.replace("__getitem__", "__cache_utils_getitem__", 1)
|
||||
exec(source)
|
||||
transformers.cache_utils.DynamicCache.__getitem__ = __cache_utils_getitem__
|
||||
pass
|
||||
# =============================================
|
||||
|
||||
# =============================================
|
||||
# Get Flash Attention v2 if Ampere (RTX 30xx, A100)
|
||||
import bitsandbytes as bnb
|
||||
|
|
@ -192,7 +216,7 @@ from transformers.models.llama.modeling_llama import logger
|
|||
# Get Xformers
|
||||
from xformers import __version__ as xformers_version
|
||||
# Temporarily disable 0.0.27 and higher - inference issues
|
||||
if Version(xformers_version) >= Version("0.0.27"):
|
||||
if False: #Version(xformers_version) >= Version("0.0.27"):
|
||||
raise ImportError(
|
||||
"Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below "\
|
||||
"then press Disconnect Runtime and then Restart it.\n"\
|
||||
|
|
@ -200,10 +224,10 @@ if Version(xformers_version) >= Version("0.0.27"):
|
|||
"%%capture\n"
|
||||
"# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n"
|
||||
'!pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n'
|
||||
'!pip install --no-deps "xformers<0.0.27" "trl<0.9.0" peft accelerate bitsandbytes\n'\
|
||||
'!pip install --no-deps "xformers<=0.0.27" trl peft accelerate bitsandbytes\n'\
|
||||
'\n'\
|
||||
f"Otherwise in local machines, your xformers version of {xformers_version} is too new.\n"\
|
||||
'Please downgrade xformers via `pip install --force-reinstall "xformers<0.0.27"'
|
||||
'Please downgrade xformers via `pip install --force-reinstall "xformers<=0.0.27"'
|
||||
)
|
||||
pass
|
||||
|
||||
|
|
@ -217,10 +241,10 @@ elif Version(torch_version) < Version("2.3.0") and Version(xformers_version) >=
|
|||
f"Unsloth: You have torch = {torch_version} but xformers = {xformers_version}.\n"\
|
||||
f"Please install xformers < 0.0.26 for torch = {torch_version}."
|
||||
)
|
||||
elif Version(torch_version) < Version("2.4.0") and Version(xformers_version) >= Version("0.0.27"):
|
||||
elif Version(torch_version) < Version("2.4.0") and Version(xformers_version) > Version("0.0.27"):
|
||||
raise ImportError(
|
||||
f"Unsloth: You have torch = {torch_version} but xformers = {xformers_version}.\n"\
|
||||
f"Please install xformers < 0.0.27 for torch = {torch_version}."
|
||||
f"Please install xformers <= 0.0.27 for torch = {torch_version}."
|
||||
)
|
||||
pass
|
||||
|
||||
|
|
@ -241,7 +265,8 @@ xformers_attention = xformers.memory_efficient_attention
|
|||
|
||||
# Check TRL version
|
||||
from trl import __version__ as trl_version
|
||||
if Version(trl_version) >= Version("0.9.0"):
|
||||
# Unsloth now supports all TRL versions!
|
||||
if False:#Version(trl_version) >= Version("0.9.0"):
|
||||
raise ImportError(
|
||||
"Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below "\
|
||||
"then press Disconnect Runtime and then Restart it.\n"\
|
||||
|
|
@ -249,13 +274,32 @@ if Version(trl_version) >= Version("0.9.0"):
|
|||
"%%capture\n"
|
||||
"# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n"
|
||||
'!pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n'
|
||||
'!pip install --no-deps "xformers<0.0.27" "trl<0.9.0" peft accelerate bitsandbytes\n'\
|
||||
'!pip install --no-deps "xformers<=0.0.27" trl peft accelerate bitsandbytes\n'\
|
||||
'\n'\
|
||||
f"Otherwise in local machines, your TRL version of {trl_version} is too new.\n"\
|
||||
'Please downgrade TRL via `pip install --force-reinstall "trl<0.9.0"'
|
||||
'Please downgrade TRL via `pip install --force-reinstall trl'
|
||||
)
|
||||
pass
|
||||
|
||||
# =============================================
|
||||
# Fix new Xformers versions TypeError: Multiple dispatch failed for 'torch._ops.aten.to.dtype_layout'
|
||||
if Version(xformers_version) >= Version("0.0.27"):
|
||||
import accelerate.utils.operations
|
||||
if hasattr(accelerate.utils.operations, "send_to_device") and \
|
||||
accelerate.utils.operations.send_to_device.__name__ != "_fixed_send_to_device":
|
||||
from accelerate.utils.operations import *
|
||||
send_to_device = inspect.getsource(accelerate.utils.operations.send_to_device)
|
||||
send_to_device = re.sub(
|
||||
r"([ ]{4,})return tensor\.to\(device\)",
|
||||
r"\1try: return tensor.to(device)\n\1except: return tensor",
|
||||
send_to_device,
|
||||
).replace("def send_to_device", "def _fixed_send_to_device")
|
||||
exec(send_to_device)
|
||||
accelerate.utils.operations.send_to_device = _fixed_send_to_device
|
||||
pass
|
||||
pass
|
||||
# =============================================
|
||||
|
||||
# =============================================
|
||||
# Torch compile settings
|
||||
|
||||
|
|
|
|||
|
|
@ -156,6 +156,7 @@ def Gemma2Attention_fast_forward(
|
|||
)
|
||||
A = A.reshape(bsz, q_len, n_heads*head_dim)
|
||||
else:
|
||||
mask = causal_mask if attention_mask is None else attention_mask
|
||||
A = slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_seq_len)
|
||||
pass
|
||||
A = self.apply_o(self, A)
|
||||
|
|
@ -413,7 +414,6 @@ def Gemma2Model_fast_forward_inference(
|
|||
SWA = attention_mask
|
||||
GA = attention_mask
|
||||
pass
|
||||
|
||||
next_decoder_cache = []
|
||||
for idx, decoder_layer in enumerate(self.model.layers):
|
||||
|
||||
|
|
|
|||
|
|
@ -419,7 +419,7 @@ pass
|
|||
def LlamaDecoderLayer_fast_forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
causal_mask: Optional[xformers.attn_bias.BlockDiagonalCausalMask] = None,
|
||||
causal_mask = None,
|
||||
attention_mask: Optional[torch.Tensor] = None,
|
||||
position_ids: Optional[torch.LongTensor] = None,
|
||||
past_key_value: Optional[Tuple[torch.Tensor]] = None,
|
||||
|
|
@ -505,7 +505,7 @@ def LlamaModel_fast_forward(
|
|||
return_dict: Optional[bool] = None,
|
||||
*args, **kwargs,
|
||||
) -> Union[Tuple, BaseModelOutputWithPast]:
|
||||
|
||||
|
||||
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
||||
assert(output_attentions is False)
|
||||
output_hidden_states = (
|
||||
|
|
@ -682,12 +682,27 @@ def LlamaModel_fast_forward(
|
|||
|
||||
|
||||
# Gemma2 has alternating SWA and global attn
|
||||
if IS_GEMMA2 and not hasattr(self, "SWA_mask"):
|
||||
if HAS_FLASH_ATTENTION_SOFTCAPPING:
|
||||
if IS_GEMMA2:
|
||||
if HAS_FLASH_ATTENTION_SOFTCAPPING and attention_mask is None:
|
||||
self.SWA_mask = True
|
||||
self.GA_mask = False
|
||||
else:
|
||||
n = self.config.max_position_embeddings
|
||||
elif attention_mask is not None:
|
||||
self.SWA_mask = _prepare_4d_causal_attention_mask_for_sdpa(
|
||||
attention_mask,
|
||||
(batch_size, seq_length),
|
||||
inputs_embeds,
|
||||
past_key_values_length,
|
||||
sliding_window = self.config.sliding_window,
|
||||
)
|
||||
self.GA_mask = _prepare_4d_causal_attention_mask_for_sdpa(
|
||||
attention_mask,
|
||||
(batch_size, seq_length),
|
||||
inputs_embeds,
|
||||
past_key_values_length,
|
||||
sliding_window = None,
|
||||
)
|
||||
elif not hasattr(self, "SWA_mask"):
|
||||
n = self.max_seq_length # self.config.max_position_embeddings
|
||||
# masked_fill is making stuff slower!
|
||||
# self. GA_mask = create_boolean_mask(n = n, sliding_window = 0)
|
||||
# self.SWA_mask = create_boolean_mask(n = n, sliding_window = self.config.sliding_window)
|
||||
|
|
@ -870,7 +885,7 @@ def CausalLM_fast_forward(fast_forward_inference):
|
|||
)
|
||||
else:
|
||||
causal_mask = xformers.attn_bias.LowerTriangularMask()
|
||||
|
||||
|
||||
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
||||
output_hidden_states = (
|
||||
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
||||
|
|
@ -879,7 +894,6 @@ def CausalLM_fast_forward(fast_forward_inference):
|
|||
|
||||
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
||||
self.model._has_no_labels = labels is None
|
||||
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
causal_mask=causal_mask,
|
||||
|
|
@ -893,7 +907,6 @@ def CausalLM_fast_forward(fast_forward_inference):
|
|||
return_dict=return_dict,
|
||||
)
|
||||
pass
|
||||
|
||||
hidden_states = outputs[0]
|
||||
bsz, q_len, hd = hidden_states.shape
|
||||
lm_head = self.lm_head.weight
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ def _get_new_mapper():
|
|||
pass
|
||||
|
||||
|
||||
def _get_model_name(model_name, load_in_4bit = True):
|
||||
def get_model_name(model_name, load_in_4bit = True):
|
||||
new_model_name = __get_model_name(
|
||||
model_name = model_name,
|
||||
load_in_4bit = load_in_4bit,
|
||||
|
|
@ -145,7 +145,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
|
||||
old_model_name = model_name
|
||||
model_name = _get_model_name(model_name, load_in_4bit)
|
||||
model_name = get_model_name(model_name, load_in_4bit)
|
||||
|
||||
# First check if it's a normal model via AutoConfig
|
||||
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled
|
||||
|
|
@ -192,7 +192,7 @@ class FastLanguageModel(FastLlamaModel):
|
|||
# Get base model for PEFT:
|
||||
if is_peft:
|
||||
# Check base model again for PEFT
|
||||
model_name = _get_model_name(peft_config.base_model_name_or_path, load_in_4bit)
|
||||
model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit)
|
||||
model_config = AutoConfig.from_pretrained(model_name, token = token, revision = revision)
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -28,12 +28,14 @@ import psutil
|
|||
import re
|
||||
from transformers.models.llama.modeling_llama import logger
|
||||
from .tokenizer_utils import fix_sentencepiece_gguf
|
||||
from huggingface_hub import HfApi
|
||||
|
||||
__all__ = [
|
||||
"print_quantization_methods",
|
||||
"unsloth_save_model",
|
||||
"save_to_gguf",
|
||||
"patch_saving_functions",
|
||||
"create_huggingface_repo",
|
||||
]
|
||||
|
||||
# Check environments
|
||||
|
|
@ -207,8 +209,9 @@ def unsloth_save_model(
|
|||
):
|
||||
if token is None and "HF_TOKEN" in os.environ:
|
||||
token = os.environ["HF_TOKEN"]
|
||||
|
||||
if token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
elif token is None and "hf_token" in os.environ:
|
||||
token = os.environ["hf_token"]
|
||||
elif token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
|
||||
if commit_message is None: commit_message = ""
|
||||
|
|
@ -555,7 +558,8 @@ def unsloth_save_model(
|
|||
logger.warning_once(f"We will save to Disk and not RAM now.")
|
||||
filename = os.path.join(temporary_location, f"{name}.pt")
|
||||
torch.save(W, filename, pickle_module = pickle, pickle_protocol = pickle.HIGHEST_PROTOCOL,)
|
||||
state_dict[name] = torch.load(filename, map_location = "cpu", mmap = True)
|
||||
# weights_only = True weirdly fails?
|
||||
state_dict[name] = torch.load(filename, map_location = "cpu", mmap = True, weights_only = False)
|
||||
pass
|
||||
for item in LLAMA_LAYERNORMS:
|
||||
try:
|
||||
|
|
@ -675,7 +679,6 @@ def unsloth_save_model(
|
|||
# Now manually go through each file and upload them manually!
|
||||
filenames = os.listdir(new_save_directory)
|
||||
|
||||
from huggingface_hub import HfApi
|
||||
hf_api = HfApi(token = save_pretrained_settings["token"])
|
||||
|
||||
print("Unsloth: Uploading all files... Please wait...")
|
||||
|
|
@ -1312,6 +1315,49 @@ def _determine_username(save_directory, old_username, token):
|
|||
pass
|
||||
|
||||
|
||||
def create_huggingface_repo(
|
||||
model,
|
||||
save_directory,
|
||||
token = None,
|
||||
private = False,
|
||||
):
|
||||
if token is None and "HF_TOKEN" in os.environ:
|
||||
token = os.environ["HF_TOKEN"]
|
||||
elif token is None and "hf_token" in os.environ:
|
||||
token = os.environ["hf_token"]
|
||||
elif token is None and "HUGGINGFACE_TOKEN" in os.environ:
|
||||
token = os.environ["HUGGINGFACE_TOKEN"]
|
||||
pass
|
||||
save_directory, username = _determine_username(save_directory, "", token)
|
||||
|
||||
from huggingface_hub import create_repo
|
||||
try:
|
||||
create_repo(
|
||||
repo_id = save_directory,
|
||||
token = token,
|
||||
repo_type = "model",
|
||||
exist_ok = False,
|
||||
private = private,
|
||||
)
|
||||
|
||||
# Create model card
|
||||
from huggingface_hub import ModelCard
|
||||
content = MODEL_CARD.format(
|
||||
username = username,
|
||||
base_model = model.config._name_or_path,
|
||||
model_type = model.config.model_type,
|
||||
method = "",
|
||||
extra = "unsloth",
|
||||
)
|
||||
card = ModelCard(content)
|
||||
card.push_to_hub(save_directory, token = token)
|
||||
except:
|
||||
pass
|
||||
hf_api = HfApi(token = token)
|
||||
return save_directory, hf_api
|
||||
pass
|
||||
|
||||
|
||||
def upload_to_huggingface(
|
||||
model,
|
||||
save_directory,
|
||||
|
|
@ -1321,6 +1367,7 @@ def upload_to_huggingface(
|
|||
file_location = None,
|
||||
old_username = None,
|
||||
private = None,
|
||||
create_config = True,
|
||||
):
|
||||
save_directory, username = _determine_username(save_directory, old_username, token)
|
||||
|
||||
|
|
@ -1350,7 +1397,6 @@ def upload_to_huggingface(
|
|||
|
||||
if file_location is not None:
|
||||
# Now upload file
|
||||
from huggingface_hub import HfApi
|
||||
hf_api = HfApi(token = token)
|
||||
|
||||
if "/" in file_location:
|
||||
|
|
@ -1372,6 +1418,8 @@ def upload_to_huggingface(
|
|||
repo_type = "model",
|
||||
commit_message = "(Trained with Unsloth)",
|
||||
)
|
||||
pass
|
||||
pass
|
||||
|
||||
hf_api.upload_file(
|
||||
path_or_fileobj = file_location,
|
||||
|
|
@ -1382,18 +1430,20 @@ def upload_to_huggingface(
|
|||
)
|
||||
|
||||
# We also upload a config.json file
|
||||
import json
|
||||
with open("_temporary_unsloth_config.json", "w") as file:
|
||||
json.dump({"model_type" : model.config.model_type}, file, indent = 4)
|
||||
if create_config:
|
||||
import json
|
||||
with open("_temporary_unsloth_config.json", "w") as file:
|
||||
json.dump({"model_type" : model.config.model_type}, file, indent = 4)
|
||||
pass
|
||||
hf_api.upload_file(
|
||||
path_or_fileobj = "_temporary_unsloth_config.json",
|
||||
path_in_repo = "config.json",
|
||||
repo_id = save_directory,
|
||||
repo_type = "model",
|
||||
commit_message = "(Trained with Unsloth)",
|
||||
)
|
||||
os.remove("_temporary_unsloth_config.json")
|
||||
pass
|
||||
hf_api.upload_file(
|
||||
path_or_fileobj = "_temporary_unsloth_config.json",
|
||||
path_in_repo = "config.json",
|
||||
repo_id = save_directory,
|
||||
repo_type = "model",
|
||||
commit_message = "(Trained with Unsloth)",
|
||||
)
|
||||
os.remove("_temporary_unsloth_config.json")
|
||||
pass
|
||||
return username
|
||||
pass
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue