diff --git a/pyproject.toml b/pyproject.toml index 59ef1b8aab..9499d771c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,15 +35,15 @@ exclude = ["images*"] huggingface = [ "packaging", "tyro", - "transformers>=4.45.1", + "transformers<4.45.0", "datasets>=2.16.0", "sentencepiece>=0.2.0", "tqdm", "psutil", "wheel>=0.42.0", "numpy", - "accelerate>=0.26.1", - "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3", + "accelerate>=0.34.1", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,<=0.11.1", "peft>=0.7.1,!=0.11.0", "protobuf<4.0.0", "huggingface_hub", @@ -212,7 +212,7 @@ colab-ampere-torch220 = [ colab-new = [ "packaging", "tyro", - "transformers>=4.45.1", + "transformers<4.45.0", "datasets>=2.16.0", "sentencepiece>=0.2.0", "tqdm", @@ -224,8 +224,8 @@ colab-new = [ "hf_transfer", ] colab-no-deps = [ - "accelerate>=0.26.1", - "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3", + "accelerate>=0.34.1", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,<=0.11.1", "peft>=0.7.1", "xformers<0.0.27", "bitsandbytes>=0.43.3", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 309c0d913e..b14bb39144 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2024.9.post3" +__version__ = "2024.9.post4" __all__ = [ "prepare_model_for_kbit_training", diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 0ac9b02743..61e8132731 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -23,6 +23,7 @@ from peft import PeftConfig, PeftModel from .mapper import INT_TO_FLOAT_MAPPER, FLOAT_TO_INT_MAPPER, MAP_TO_UNSLOTH_16bit import os from huggingface_hub.utils._token import get_token +from huggingface_hub import HfFileSystem # https://github.com/huggingface/transformers/pull/26037 allows 4 bit loading! from packaging.version import Version @@ -191,14 +192,29 @@ class FastLanguageModel(FastLlamaModel): is_peft = False pass - # Cannot be both! - if (is_model and is_peft) and not SUPPORTS_LLAMA32: + # Both config.json and adapter_config.json should not exist! + + # Old transformers versions check + both_exist = (is_model and is_peft) and not SUPPORTS_LLAMA32 + + if SUPPORTS_LLAMA32: + # New transformers need to check manually. + files = HfFileSystem(token = token).glob(os.path.join(model_name, "*.json")) + files = (os.path.split(x)[-1] for x in files) + if sum(x == "adapter_config.json" or x == "config.json" for x in files) >= 2: + both_exist = True + pass + pass + + # Error out if both LoRA and normal model config exists. + if both_exist: raise RuntimeError( "Unsloth: Your repo has a LoRA adapter and a base model.\n"\ "You have 2 files `config.json` and `adapter_config.json`.\n"\ "We must only allow one config file.\n"\ "Please separate the LoRA and base models to 2 repos." ) + elif not is_model and not is_peft: error = autoconfig_error or peft_error # Old transformers version diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 988fda4938..0b8c08a371 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -18,8 +18,14 @@ from ..kernels import patch_rms_layernorm, unpatch_rms_layernorm from ..kernels import patch_llama_for_causal_lm, unpatch_llama_for_causal_lm from ._utils import patch_gradient_checkpointing -from transformers import AutoProcessor, AutoModelForVision2Seq - +from transformers import AutoProcessor +try: + from transformers import MllamaForConditionalGeneration +except: + raise ImportError( + "Unsloth: Please update your transformers version to 4.46.0 for Llama 3.2 support!" + ) +pass class FastVisionModel: @@ -56,7 +62,6 @@ class FastVisionModel: ) pass if token is None: token = get_token() - if model_patcher is None: model_patcher = FastLlamaModel SUPPORTS_BFLOAT16 = is_bfloat16_supported() gpu_stats = torch.cuda.get_device_properties(0) max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3) @@ -87,48 +92,6 @@ class FastVisionModel: assert(dtype == torch.float16 or dtype == torch.bfloat16 or dtype == torch.float32) - # RoPE Scaling - model_config = AutoConfig.from_pretrained(model_name, token = token) - model_max_seq_length = model_config.max_position_embeddings - - # Check if RoPE Scaling is even allowed - model_function = MODEL_FOR_CAUSAL_LM_MAPPING[model_config.__class__] - has_rope_scaling = False - try: - with open(inspect.getfile(model_function), "r") as file: - has_rope_scaling = "self.config.rope_scaling" in file.read() - except: pass - has_rope_scaling = True - - # If max_seq_length is not specified, use maximum fron config - if max_seq_length is None: - max_seq_length = model_max_seq_length - pass - - if (rope_scaling is None) and (max_seq_length > model_max_seq_length): - - rope_scaling = max_seq_length / model_max_seq_length - - logger.warning_once( - f"Unsloth: {model_name} can only handle sequence lengths of at most "\ - f"{model_max_seq_length}.\nBut with kaiokendev's RoPE scaling of "\ - f"{round(rope_scaling, 3)}, it can be magically be extended to "\ - f"{max_seq_length}!" - ) - - # Warn RoPE scaling isn't allowed - if not has_rope_scaling: - raise RuntimeError( - "However, {model_name} doesn't support RoPE Scaling!\n"\ - "Please file a feature request at https://github.com/unslothai/unsloth." - ) - pass - - rope_scaling = {"type": "linear", "factor": rope_scaling,} - - # Add to kwargs - kwargs["rope_scaling"] = rope_scaling - pass # We currently only support NVIDIA GPUs - AMD / Intel is a work in progress! pre_check = check_nvidia() @@ -142,16 +105,11 @@ class FastVisionModel: ) pass - # https://huggingface.co/togethercomputer/LLaMA-2-7B-32K/discussions/12 - # RoPE Scaling's max_position_embeddings must be updated - max_position_embeddings = max(max_seq_length, model_max_seq_length) - kwargs.pop("attn_implementation", None); # No need since we auto call it - # Cannot be None, since HF now checks for the config if load_in_4bit: kwargs["quantization_config"] = bnb_config self.pre_patch() - model = AutoModelForVision2Seq.from_pretrained( + model = MllamaForConditionalGeneration.from_pretrained( model_name, device_map = device_map, torch_dtype = dtype, @@ -159,7 +117,7 @@ class FastVisionModel: token = token, max_position_embeddings = max_position_embeddings, trust_remote_code = trust_remote_code, - attn_implementation = "eager", + attn_implementation = "sdpa", **kwargs, ) self.post_unpatch() diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 04690d3566..cdce372b50 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -1135,7 +1135,39 @@ from inspect import getsource import trl.trainer.sft_trainer from trl.trainer.sft_trainer import * from transformers.trainer import * -from trl.trainer.sft_trainer import neftune_post_forward_hook +try: + from trl.trainer.sft_trainer import neftune_post_forward_hook +except: + def neftune_post_forward_hook(module, input, output): + """ + Implements the NEFTune forward pass for the model using forward hooks. Note this works only for + torch.nn.Embedding layers. This method is slightly adapted from the original source code + that can be found here: https://github.com/neelsjain/NEFTune + + Simply add it to your model as follows: + ```python + model = ... + model.embed_tokens.neftune_noise_alpha = 0.1 + model.embed_tokens.register_forward_hook(neftune_post_forward_hook) + ``` + + Args: + module (`torch.nn.Module`): + The embedding module where the hook is attached. Note that you need to set + `module.neftune_noise_alpha` to the desired noise alpha value. + input (`torch.Tensor`): + The input tensor to the model. + output (`torch.Tensor`): + The output tensor of the model (i.e. the embeddings). + """ + if module.training: + dims = torch.tensor(output.size(1) * output.size(2)) + mag_norm = module.neftune_noise_alpha / torch.sqrt(dims) + output = output + torch.zeros_like(output).uniform_(-mag_norm, mag_norm) + return output + pass +pass + def patch_sft_trainer_tokenizer(): """