diff --git a/pyproject.toml b/pyproject.toml index 2cbe68f4a6..b61908a690 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,111 +43,154 @@ huggingface = [ "wheel>=0.42.0", "numpy", "accelerate>=0.26.1", - "trl>=0.7.9,<0.9.0", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3", "peft>=0.7.1,!=0.11.0", "protobuf<4.0.0", "huggingface_hub", "hf_transfer", ] cu118only = [ - "xformers==0.0.22.post7", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu121only = [ - "xformers==0.0.22.post7", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.22.post7-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu118onlytorch211 = [ - "xformers==0.0.23", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu121onlytorch211 = [ - "xformers==0.0.23", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu118onlytorch212 = [ - "xformers==0.0.23.post1", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.23.post1%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu121onlytorch212 = [ - "xformers==0.0.23.post1", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.23.post1-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu118onlytorch220 = [ - "xformers==0.0.24", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.24%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu121onlytorch220 = [ - "xformers==0.0.24", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.24-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", ] cu118onlytorch230 = [ - "xformers==0.0.26.post1", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'", ] cu121onlytorch230 = [ - "xformers==0.0.26.post1", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'", +] +cu118onlytorch240 = [ + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", + "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.27.post2%2Bcu118-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'", +] +cu121onlytorch240 = [ + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp310-cp310-manylinux2014_x86_64.whl ; python_version=='3.10'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp311-cp311-manylinux2014_x86_64.whl ; python_version=='3.11'", + "xformers @ https://download.pytorch.org/whl/cu121/xformers-0.0.27.post2-cp312-cp312-manylinux2014_x86_64.whl ; python_version=='3.12'", ] - cu118 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118only]", ] cu121 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121only]", ] cu118-torch211 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118onlytorch211]", ] cu121-torch211 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch211]", ] cu118-torch212 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118onlytorch212]", ] cu121-torch212 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch212]", ] cu118-torch220 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118onlytorch220]", ] cu121-torch220 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch220]", ] cu118-torch230 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118onlytorch230]", ] cu121-torch230 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch230]", ] +cu118-torch240 = [ + "unsloth[huggingface]", + "bitsandbytes>=0.43.3", + "unsloth[cu118onlytorch240]", +] +cu121-torch240 = [ + "unsloth[huggingface]", + "bitsandbytes>=0.43.3", + "unsloth[cu121onlytorch240]", +] kaggle = [ "unsloth[huggingface]", ] kaggle-new = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", ] conda = [ "unsloth[huggingface]", ] colab-torch211 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch211]", ] colab-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch211]", "packaging", "ninja", @@ -155,12 +198,12 @@ colab-ampere-torch211 = [ ] colab-torch220 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch220]", ] colab-ampere-torch220 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch220]", "packaging", "ninja", @@ -182,10 +225,10 @@ colab-new = [ ] colab-no-deps = [ "accelerate>=0.26.1", - "trl>=0.7.9", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3", "peft>=0.7.1", "xformers<0.0.27", - "bitsandbytes", + "bitsandbytes>=0.43.3", "protobuf<4.0.0", ] colab = [ @@ -199,7 +242,7 @@ colab-ampere = [ ] cu118-ampere = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118only]", "packaging", "ninja", @@ -207,7 +250,7 @@ cu118-ampere = [ ] cu121-ampere = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121only]", "packaging", "ninja", @@ -215,7 +258,7 @@ cu121-ampere = [ ] cu118-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118onlytorch211]", "packaging", "ninja", @@ -223,7 +266,7 @@ cu118-ampere-torch211 = [ ] cu121-ampere-torch211 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch211]", "packaging", "ninja", @@ -231,7 +274,7 @@ cu121-ampere-torch211 = [ ] cu118-ampere-torch220 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118onlytorch220]", "packaging", "ninja", @@ -239,7 +282,7 @@ cu118-ampere-torch220 = [ ] cu121-ampere-torch220 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch220]", "packaging", "ninja", @@ -247,7 +290,7 @@ cu121-ampere-torch220 = [ ] cu118-ampere-torch230 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu118onlytorch230]", "packaging", "ninja", @@ -255,12 +298,28 @@ cu118-ampere-torch230 = [ ] cu121-ampere-torch230 = [ "unsloth[huggingface]", - "bitsandbytes", + "bitsandbytes>=0.43.3", "unsloth[cu121onlytorch230]", "packaging", "ninja", "flash-attn>=2.6.3", ] +cu118-ampere-torch240 = [ + "unsloth[huggingface]", + "bitsandbytes>=0.43.3", + "unsloth[cu118onlytorch240]", + "packaging", + "ninja", + "flash-attn>=2.6.3", +] +cu121-ampere-torch240 = [ + "unsloth[huggingface]", + "bitsandbytes>=0.43.3", + "unsloth[cu121onlytorch240]", + "packaging", + "ninja", + "flash-attn>=2.6.3", +] [project.urls] homepage = "http://www.unsloth.ai" diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 195fd5bb65..0c0057496b 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -53,7 +53,9 @@ from packaging.version import Version # Disable some warnings which can get annoying warnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch") warnings.filterwarnings(action = "ignore", category = UserWarning, module = "huggingface_hub") +warnings.filterwarnings(action = "ignore", category = UserWarning, module = "trl") warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "huggingface_hub") +warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "xformers") warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "subprocess") warnings.filterwarnings(action = "ignore", category = UserWarning, module = "transformers") warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "accelerate") @@ -133,6 +135,28 @@ else: pass # ============================================= +# ============================================= +# Fix KeyError: 'Cache only has 0 layers, attempted to access layer with index 0' +import transformers.cache_utils +if hasattr(transformers.cache_utils, "DynamicCache") and \ + transformers.cache_utils.DynamicCache.__getitem__.__name__ != "__cache_utils_getitem__": + + source = inspect.getsource(transformers.cache_utils.DynamicCache.__getitem__) + start = source.find("def") + spaces = start*" " + source = source.split("\n") + source = "\n".join(x[start:] for x in source) + where = source.find("raise KeyError") + source = source[:where] + \ + f"if len(self) == 0:\n{spaces}{spaces}"\ + " raise RuntimeError('Unsloth: You must call `FastLanguageModel.for_inference(model)` before doing inference for Unsloth models.')\n" + \ + f"{spaces}{spaces}else:\n{spaces}{spaces}{spaces}" + source[where:] + source = source.replace("__getitem__", "__cache_utils_getitem__", 1) + exec(source) + transformers.cache_utils.DynamicCache.__getitem__ = __cache_utils_getitem__ +pass +# ============================================= + # ============================================= # Get Flash Attention v2 if Ampere (RTX 30xx, A100) import bitsandbytes as bnb @@ -192,7 +216,7 @@ from transformers.models.llama.modeling_llama import logger # Get Xformers from xformers import __version__ as xformers_version # Temporarily disable 0.0.27 and higher - inference issues -if Version(xformers_version) >= Version("0.0.27"): +if False: #Version(xformers_version) >= Version("0.0.27"): raise ImportError( "Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below "\ "then press Disconnect Runtime and then Restart it.\n"\ @@ -200,10 +224,10 @@ if Version(xformers_version) >= Version("0.0.27"): "%%capture\n" "# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n" '!pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n' - '!pip install --no-deps "xformers<0.0.27" "trl<0.9.0" peft accelerate bitsandbytes\n'\ + '!pip install --no-deps "xformers<=0.0.27" trl peft accelerate bitsandbytes\n'\ '\n'\ f"Otherwise in local machines, your xformers version of {xformers_version} is too new.\n"\ - 'Please downgrade xformers via `pip install --force-reinstall "xformers<0.0.27"' + 'Please downgrade xformers via `pip install --force-reinstall "xformers<=0.0.27"' ) pass @@ -217,10 +241,10 @@ elif Version(torch_version) < Version("2.3.0") and Version(xformers_version) >= f"Unsloth: You have torch = {torch_version} but xformers = {xformers_version}.\n"\ f"Please install xformers < 0.0.26 for torch = {torch_version}." ) -elif Version(torch_version) < Version("2.4.0") and Version(xformers_version) >= Version("0.0.27"): +elif Version(torch_version) < Version("2.4.0") and Version(xformers_version) > Version("0.0.27"): raise ImportError( f"Unsloth: You have torch = {torch_version} but xformers = {xformers_version}.\n"\ - f"Please install xformers < 0.0.27 for torch = {torch_version}." + f"Please install xformers <= 0.0.27 for torch = {torch_version}." ) pass @@ -241,7 +265,8 @@ xformers_attention = xformers.memory_efficient_attention # Check TRL version from trl import __version__ as trl_version -if Version(trl_version) >= Version("0.9.0"): +# Unsloth now supports all TRL versions! +if False:#Version(trl_version) >= Version("0.9.0"): raise ImportError( "Unsloth: If you are in Colab, we updated the top cell install instructions - please change it to below "\ "then press Disconnect Runtime and then Restart it.\n"\ @@ -249,13 +274,32 @@ if Version(trl_version) >= Version("0.9.0"): "%%capture\n" "# Installs Unsloth, Xformers (Flash Attention) and all other packages!\n" '!pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"\n' - '!pip install --no-deps "xformers<0.0.27" "trl<0.9.0" peft accelerate bitsandbytes\n'\ + '!pip install --no-deps "xformers<=0.0.27" trl peft accelerate bitsandbytes\n'\ '\n'\ f"Otherwise in local machines, your TRL version of {trl_version} is too new.\n"\ - 'Please downgrade TRL via `pip install --force-reinstall "trl<0.9.0"' + 'Please downgrade TRL via `pip install --force-reinstall trl' ) pass +# ============================================= +# Fix new Xformers versions TypeError: Multiple dispatch failed for 'torch._ops.aten.to.dtype_layout' +if Version(xformers_version) >= Version("0.0.27"): + import accelerate.utils.operations + if hasattr(accelerate.utils.operations, "send_to_device") and \ + accelerate.utils.operations.send_to_device.__name__ != "_fixed_send_to_device": + from accelerate.utils.operations import * + send_to_device = inspect.getsource(accelerate.utils.operations.send_to_device) + send_to_device = re.sub( + r"([ ]{4,})return tensor\.to\(device\)", + r"\1try: return tensor.to(device)\n\1except: return tensor", + send_to_device, + ).replace("def send_to_device", "def _fixed_send_to_device") + exec(send_to_device) + accelerate.utils.operations.send_to_device = _fixed_send_to_device + pass +pass +# ============================================= + # ============================================= # Torch compile settings diff --git a/unsloth/models/gemma2.py b/unsloth/models/gemma2.py index 1cbaf5b169..ea9f53e7db 100644 --- a/unsloth/models/gemma2.py +++ b/unsloth/models/gemma2.py @@ -156,6 +156,7 @@ def Gemma2Attention_fast_forward( ) A = A.reshape(bsz, q_len, n_heads*head_dim) else: + mask = causal_mask if attention_mask is None else attention_mask A = slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, kv_seq_len) pass A = self.apply_o(self, A) @@ -413,7 +414,6 @@ def Gemma2Model_fast_forward_inference( SWA = attention_mask GA = attention_mask pass - next_decoder_cache = [] for idx, decoder_layer in enumerate(self.model.layers): diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index e300e07e0c..2a07da6ce5 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -419,7 +419,7 @@ pass def LlamaDecoderLayer_fast_forward( self, hidden_states: torch.Tensor, - causal_mask: Optional[xformers.attn_bias.BlockDiagonalCausalMask] = None, + causal_mask = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.LongTensor] = None, past_key_value: Optional[Tuple[torch.Tensor]] = None, @@ -505,7 +505,7 @@ def LlamaModel_fast_forward( return_dict: Optional[bool] = None, *args, **kwargs, ) -> Union[Tuple, BaseModelOutputWithPast]: - + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions assert(output_attentions is False) output_hidden_states = ( @@ -682,12 +682,27 @@ def LlamaModel_fast_forward( # Gemma2 has alternating SWA and global attn - if IS_GEMMA2 and not hasattr(self, "SWA_mask"): - if HAS_FLASH_ATTENTION_SOFTCAPPING: + if IS_GEMMA2: + if HAS_FLASH_ATTENTION_SOFTCAPPING and attention_mask is None: self.SWA_mask = True self.GA_mask = False - else: - n = self.config.max_position_embeddings + elif attention_mask is not None: + self.SWA_mask = _prepare_4d_causal_attention_mask_for_sdpa( + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, + sliding_window = self.config.sliding_window, + ) + self.GA_mask = _prepare_4d_causal_attention_mask_for_sdpa( + attention_mask, + (batch_size, seq_length), + inputs_embeds, + past_key_values_length, + sliding_window = None, + ) + elif not hasattr(self, "SWA_mask"): + n = self.max_seq_length # self.config.max_position_embeddings # masked_fill is making stuff slower! # self. GA_mask = create_boolean_mask(n = n, sliding_window = 0) # self.SWA_mask = create_boolean_mask(n = n, sliding_window = self.config.sliding_window) @@ -870,7 +885,7 @@ def CausalLM_fast_forward(fast_forward_inference): ) else: causal_mask = xformers.attn_bias.LowerTriangularMask() - + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states @@ -879,7 +894,6 @@ def CausalLM_fast_forward(fast_forward_inference): # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) self.model._has_no_labels = labels is None - outputs = self.model( input_ids=input_ids, causal_mask=causal_mask, @@ -893,7 +907,6 @@ def CausalLM_fast_forward(fast_forward_inference): return_dict=return_dict, ) pass - hidden_states = outputs[0] bsz, q_len, hd = hidden_states.shape lm_head = self.lm_head.weight diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 47152d6764..cce22aebf6 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -93,7 +93,7 @@ def _get_new_mapper(): pass -def _get_model_name(model_name, load_in_4bit = True): +def get_model_name(model_name, load_in_4bit = True): new_model_name = __get_model_name( model_name = model_name, load_in_4bit = load_in_4bit, @@ -145,7 +145,7 @@ class FastLanguageModel(FastLlamaModel): token = os.environ["HUGGINGFACE_TOKEN"] old_model_name = model_name - model_name = _get_model_name(model_name, load_in_4bit) + model_name = get_model_name(model_name, load_in_4bit) # First check if it's a normal model via AutoConfig from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled @@ -192,7 +192,7 @@ class FastLanguageModel(FastLlamaModel): # Get base model for PEFT: if is_peft: # Check base model again for PEFT - model_name = _get_model_name(peft_config.base_model_name_or_path, load_in_4bit) + model_name = get_model_name(peft_config.base_model_name_or_path, load_in_4bit) model_config = AutoConfig.from_pretrained(model_name, token = token, revision = revision) pass diff --git a/unsloth/save.py b/unsloth/save.py index a5904efc1b..f45d8062ad 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -28,12 +28,14 @@ import psutil import re from transformers.models.llama.modeling_llama import logger from .tokenizer_utils import fix_sentencepiece_gguf +from huggingface_hub import HfApi __all__ = [ "print_quantization_methods", "unsloth_save_model", "save_to_gguf", "patch_saving_functions", + "create_huggingface_repo", ] # Check environments @@ -207,8 +209,9 @@ def unsloth_save_model( ): if token is None and "HF_TOKEN" in os.environ: token = os.environ["HF_TOKEN"] - - if token is None and "HUGGINGFACE_TOKEN" in os.environ: + elif token is None and "hf_token" in os.environ: + token = os.environ["hf_token"] + elif token is None and "HUGGINGFACE_TOKEN" in os.environ: token = os.environ["HUGGINGFACE_TOKEN"] if commit_message is None: commit_message = "" @@ -555,7 +558,8 @@ def unsloth_save_model( logger.warning_once(f"We will save to Disk and not RAM now.") filename = os.path.join(temporary_location, f"{name}.pt") torch.save(W, filename, pickle_module = pickle, pickle_protocol = pickle.HIGHEST_PROTOCOL,) - state_dict[name] = torch.load(filename, map_location = "cpu", mmap = True) + # weights_only = True weirdly fails? + state_dict[name] = torch.load(filename, map_location = "cpu", mmap = True, weights_only = False) pass for item in LLAMA_LAYERNORMS: try: @@ -675,7 +679,6 @@ def unsloth_save_model( # Now manually go through each file and upload them manually! filenames = os.listdir(new_save_directory) - from huggingface_hub import HfApi hf_api = HfApi(token = save_pretrained_settings["token"]) print("Unsloth: Uploading all files... Please wait...") @@ -1312,6 +1315,49 @@ def _determine_username(save_directory, old_username, token): pass +def create_huggingface_repo( + model, + save_directory, + token = None, + private = False, +): + if token is None and "HF_TOKEN" in os.environ: + token = os.environ["HF_TOKEN"] + elif token is None and "hf_token" in os.environ: + token = os.environ["hf_token"] + elif token is None and "HUGGINGFACE_TOKEN" in os.environ: + token = os.environ["HUGGINGFACE_TOKEN"] + pass + save_directory, username = _determine_username(save_directory, "", token) + + from huggingface_hub import create_repo + try: + create_repo( + repo_id = save_directory, + token = token, + repo_type = "model", + exist_ok = False, + private = private, + ) + + # Create model card + from huggingface_hub import ModelCard + content = MODEL_CARD.format( + username = username, + base_model = model.config._name_or_path, + model_type = model.config.model_type, + method = "", + extra = "unsloth", + ) + card = ModelCard(content) + card.push_to_hub(save_directory, token = token) + except: + pass + hf_api = HfApi(token = token) + return save_directory, hf_api +pass + + def upload_to_huggingface( model, save_directory, @@ -1321,6 +1367,7 @@ def upload_to_huggingface( file_location = None, old_username = None, private = None, + create_config = True, ): save_directory, username = _determine_username(save_directory, old_username, token) @@ -1350,7 +1397,6 @@ def upload_to_huggingface( if file_location is not None: # Now upload file - from huggingface_hub import HfApi hf_api = HfApi(token = token) if "/" in file_location: @@ -1372,6 +1418,8 @@ def upload_to_huggingface( repo_type = "model", commit_message = "(Trained with Unsloth)", ) + pass + pass hf_api.upload_file( path_or_fileobj = file_location, @@ -1382,18 +1430,20 @@ def upload_to_huggingface( ) # We also upload a config.json file - import json - with open("_temporary_unsloth_config.json", "w") as file: - json.dump({"model_type" : model.config.model_type}, file, indent = 4) + if create_config: + import json + with open("_temporary_unsloth_config.json", "w") as file: + json.dump({"model_type" : model.config.model_type}, file, indent = 4) + pass + hf_api.upload_file( + path_or_fileobj = "_temporary_unsloth_config.json", + path_in_repo = "config.json", + repo_id = save_directory, + repo_type = "model", + commit_message = "(Trained with Unsloth)", + ) + os.remove("_temporary_unsloth_config.json") pass - hf_api.upload_file( - path_or_fileobj = "_temporary_unsloth_config.json", - path_in_repo = "config.json", - repo_id = save_directory, - repo_type = "model", - commit_message = "(Trained with Unsloth)", - ) - os.remove("_temporary_unsloth_config.json") pass return username pass