diff --git a/pyproject.toml b/pyproject.toml index 9c862a2617..e6f663a969 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ huggingface = [ "accelerate>=0.26.1", "trl>=0.7.9", "peft>=0.7.1", + "protobuf<4.0.0", ] cu118only = [ "xformers @ https://download.pytorch.org/whl/cu118/xformers-0.0.22.post7%2Bcu118-cp39-cp39-manylinux2014_x86_64.whl ; python_version=='3.9'", @@ -170,6 +171,7 @@ colab-new = [ "psutil", "wheel>=0.42.0", "numpy", + "protobuf<4.0.0", ] colab-no-deps = [ "accelerate>=0.26.1", @@ -177,6 +179,7 @@ colab-no-deps = [ "peft>=0.7.1", "xformers", "bitsandbytes", + "protobuf<4.0.0", ] colab = [ "unsloth[cu121]", diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index a7ade9fc32..f39d34fa35 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -1068,17 +1068,39 @@ class FastLlamaModel: # https://huggingface.co/togethercomputer/LLaMA-2-7B-32K/discussions/12 # RoPE Scaling's max_position_embeddings must be updated max_position_embeddings = max(max_seq_length, model_max_seq_length) - model = AutoModelForCausalLM.from_pretrained( - model_name, - device_map = device_map, - torch_dtype = dtype, - quantization_config = bnb_config, - token = token, - rope_scaling = rope_scaling, - max_position_embeddings = max_position_embeddings, - trust_remote_code = trust_remote_code, - **kwargs, - ) + try: + model = AutoModelForCausalLM.from_pretrained( + model_name, + device_map = device_map, + torch_dtype = dtype, + quantization_config = bnb_config, + token = token, + rope_scaling = rope_scaling, + max_position_embeddings = max_position_embeddings, + trust_remote_code = trust_remote_code, + **kwargs, + ) + except Exception as error: + if "rope_scaling" in str(error): + if rope_scaling is not None: + raise TypeError("Unsloth: {model_name} does not support rope_scaling.") + pass + + # Counteract missing rope_scaling + model = AutoModelForCausalLM.from_pretrained( + model_name, + device_map = device_map, + torch_dtype = dtype, + quantization_config = bnb_config, + token = token, + max_position_embeddings = max_position_embeddings, + trust_remote_code = trust_remote_code, + **kwargs, + ) + else: + raise error + pass + pass # Counteract saved tokenizers tokenizer_name = model_name if tokenizer_name is None else tokenizer_name diff --git a/unsloth/save.py b/unsloth/save.py index 49d88bffc0..636a3d84e7 100644 --- a/unsloth/save.py +++ b/unsloth/save.py @@ -684,7 +684,7 @@ pass def install_llama_cpp_make_non_blocking(): - env = { **os.environ, "LLAMA_CUBLAS": "1", } + env = { **os.environ, "LLAMA_CUDA": "1", } n_jobs = max(int(psutil.cpu_count()*1.5), 1) # Force make clean os.system("make clean -C llama.cpp") @@ -752,7 +752,7 @@ pass def install_llama_cpp_blocking(use_cuda = True): - use_cuda = "LLAMA_CUBLAS=1" if use_cuda else "" + use_cuda = "LLAMA_CUDA=1" if use_cuda else "" commands = [ "git clone https://github.com/ggerganov/llama.cpp", @@ -937,7 +937,7 @@ def save_to_gguf( "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ "You must run this in the same folder as you're saving your model.\n"\ "git clone https://github.com/ggerganov/llama.cpp\n"\ - "cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\ + "cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\ "Once that's done, redo the quantization." ) pass @@ -966,7 +966,7 @@ def save_to_gguf( "You do not need to close this Python program. Run the following commands in a new terminal:\n"\ "You must run this in the same folder as you're saving your model.\n"\ "git clone https://github.com/ggerganov/llama.cpp\n"\ - "cd llama.cpp && make clean && LLAMA_CUBLAS=1 make all -j\n"\ + "cd llama.cpp && make clean && LLAMA_CUDA=1 make all -j\n"\ "Once that's done, redo the quantization." ) pass diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index 46de1c98ad..2640f321d9 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -31,6 +31,12 @@ IGNORED_TOKENIZER_CHECKING = frozenset(( "CodeLlamaTokenizer", )) +# Check environments +keynames = "\n" + "\n".join(os.environ.keys()) +IS_COLAB_ENVIRONMENT = "\nCOLAB_" in keynames +IS_KAGGLE_ENVIRONMENT = "\nKAGGLE_" in keynames +del keynames + def try_fix_tokenizer(tokenizer, prepend = True): @@ -179,10 +185,19 @@ def assert_same_tokenization(slow_tokenizer, fast_tokenizer): if x.endswith("_token") and x.count("_") == 1 ))) all_special_tokens = list(set(special_tokens + slow_tokenizer.all_special_tokens)) - string = "\n".join(all_special_tokens) + \ - "A quick brown fox jumps over the lazy dog!!\n\n" + \ - "".join(all_special_tokens) - return slow_tokenizer(string).input_ids == fast_tokenizer(string).input_ids + try: + string = "\n".join(all_special_tokens) + \ + "A quick brown fox jumps over the lazy dog!!\n\n" + \ + "".join(all_special_tokens) + return slow_tokenizer(string).input_ids == fast_tokenizer(string).input_ids + except: + # For eg see https://github.com/unslothai/unsloth/issues/292 + # Sometimes tokenizer has weird tokens, causing a combined tokenization to fail. + # [TODO] We temporarily disable this for CodeLlama tokenizers + if slow_tokenizer.__repr__().split("(", 1)[0] in IGNORED_TOKENIZER_CHECKING: + return True + else: + return False pass @@ -203,7 +218,6 @@ def fix_sentencepiece_tokenizer( # First save the old tokenizer old_tokenizer.save_pretrained(temporary_location) - from sentencepiece import SentencePieceProcessor tokenizer_file = sentencepiece_model_pb2.ModelProto() tokenizer_file.ParseFromString(open(f"{temporary_location}/tokenizer.model", "rb").read()) @@ -220,7 +234,11 @@ def fix_sentencepiece_tokenizer( continue pass ids = ids[0] - tokenizer_piece = tokenizer_file.pieces[ids] + # [TODO] Hack for Starling - try except + try: + tokenizer_piece = tokenizer_file.pieces[ids] + except: + continue assert(tokenizer_piece.piece == old_token) tokenizer_piece.piece = new_token pass @@ -243,7 +261,14 @@ def load_correct_tokenizer( padding_side = "right", token = None, trust_remote_code = False, + cache_dir = "huggingface_tokenizers_cache", ): + if IS_COLAB_ENVIRONMENT or IS_KAGGLE_ENVIRONMENT: + cache_dir = cache_dir + else: + cache_dir = None + pass + slow_tokenizer = AutoTokenizer.from_pretrained( tokenizer_name, model_max_length = model_max_length, @@ -251,6 +276,7 @@ def load_correct_tokenizer( token = token, trust_remote_code = trust_remote_code, use_fast = False, + cache_dir = cache_dir, ) fast_tokenizer = AutoTokenizer.from_pretrained( tokenizer_name, @@ -258,6 +284,7 @@ def load_correct_tokenizer( padding_side = padding_side, token = token, trust_remote_code = trust_remote_code, + cache_dir = cache_dir, ) fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token fast_tokenizer.add_eos_token = slow_tokenizer.add_eos_token @@ -375,6 +402,12 @@ def check_tokenizer( ) pass + if IS_COLAB_ENVIRONMENT or IS_KAGGLE_ENVIRONMENT: + cache_dir = "huggingface_tokenizers_cache" + else: + cache_dir = None + pass + # Try slow tokenizer which can fix things! tokenizer = AutoTokenizer.from_pretrained( model_name, @@ -382,6 +415,7 @@ def check_tokenizer( padding_side = padding_side, token = token, use_fast = False, + cache_dir = cache_dir, ) return check_tokenizer( model = model,