From 28e9de27d7f16b25ab72eef2bcf29c16f05a905a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Sat, 6 Jul 2024 18:50:00 -0700 Subject: [PATCH] Debugging (#739) * Update gemma2.py * Update llama.py * Update llama.py * Update gemma2.py * init * Update gemma2.py * Update gemma2.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update gemma2.py * Update gemma2.py * Update gemma2.py * All RoPE Scaling support * cleanup * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * exec * exec * Attention_Module * attention_module * imports * exec * Update llama.py * Update llama.py * boolean mask * revert masking * Update llama.py * Update save.py * Update llama.py * Update gemma2.py * Update gemma2.py * Update gemma2.py * Update utils.py * retry * Update gemma2.py * Update gemma2.py * Update gemma2.py * Update _utils.py * Update _utils.py * Update gemma2.py * Update chat_templates.py * Gemma 2 Ollama support * Update llama.py * Update llama.py * error handling * Update _utils.py * Update _utils.py * Stats for debugging * Update _utils.py * Update _utils.py * Debugging * Update tokenizer_utils.py * Update _utils.py --- unsloth/__init__.py | 5 +--- unsloth/models/_utils.py | 39 ++++++++++++++++++------- unsloth/models/llama.py | 59 +++++++++++++++----------------------- unsloth/tokenizer_utils.py | 34 +++++++++++++++------- 4 files changed, 76 insertions(+), 61 deletions(-) diff --git a/unsloth/__init__.py b/unsloth/__init__.py index 298ed13399..feb550be5f 100644 --- a/unsloth/__init__.py +++ b/unsloth/__init__.py @@ -39,10 +39,7 @@ if "CUDA_VISIBLE_DEVICES" in os.environ: first_id = devices.split(",")[0] warnings.warn( f"Unsloth: 'CUDA_VISIBLE_DEVICES' is currently {devices} \n"\ - "Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so "\ - "enabling it will require much more work, so we have to prioritize. Please understand!"\ - "We do have a beta version, which you can contact us about!\n"\ - "Thank you for your understanding and we appreciate it immensely!\n\n"\ + "Unsloth currently does not support multi GPU setups - but we are working on it!\n"\ "Multiple CUDA devices detected but we require a single device.\n"\ f"We will override CUDA_VISIBLE_DEVICES to first device: {first_id}." ) diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index fd1b87c532..e0df218a93 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -21,6 +21,7 @@ __all__ = [ "xformers_version", "__version__", "HAS_FLASH_ATTENTION", + "PRE_CHECK", "platform_system", "patch_tokenizer", "get_statistics", @@ -32,30 +33,27 @@ __all__ = [ "unsloth_offloaded_gradient_checkpoint", "torch_compile_options", "patch_linear_scaling", + "check_nvidia", "create_boolean_mask", ] import torch from typing import Union, Optional, List, Any, Callable, Tuple -import warnings from platform import system as platform_system platform_system = platform_system() -import math import numpy as np -import os -import psutil -import inspect -import re +import warnings, subprocess, re, inspect, psutil, os, math # ============================================= # Disable some warnings which can get annoying warnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch") warnings.filterwarnings(action = "ignore", category = UserWarning, module = "huggingface_hub") +warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "huggingface_hub") warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "subprocess") warnings.filterwarnings(action = "ignore", category = UserWarning, module = "transformers") warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "accelerate") -warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "huggingface_hub") warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "multiprocessing") +warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "multiprocess") # Stop "Special tokens have been added in the vocabulary, ..." import logging @@ -74,7 +72,10 @@ for model_name in model_architectures: config_filename = f"{model_name.title()}Config" exec(f"from {config_filepath} import {config_filename}", globals()) - config = inspect.getsource(eval(config_filename)) + try: + config = inspect.getsource(eval(config_filename)) + except: + continue if "rope_scaling" in config: continue config = re.sub( r"(\*\*kwargs)[\s]{0,}\,[\s]{0,}\)[\s]{0,}\:", @@ -345,7 +346,6 @@ def get_statistics(): # We simply download a README.md file from HF - all data is made public. # This is simply so we can check if some envs are broken or not. try: - from huggingface_hub import hf_hub_download from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled import psutil n_cpus = psutil.cpu_count(logical = False) @@ -367,7 +367,13 @@ def get_statistics(): disable_progress_bars() disabled = True pass - hf_hub_download(f"unslothai/statistics-{statistics}", "README.md", force_download = True) + + from transformers import AutoModelForCausalLM + stats_model = AutoModelForCausalLM.from_pretrained( + f"unslothai/statistics-{statistics}", + force_download = True, + ) + del stats_model if disabled: enable_progress_bars() pass @@ -659,6 +665,19 @@ def patch_linear_scaling( pass +def check_nvidia(): + # Unsloth doesn't work yet on AMD devices - we're working on it! + try: + output = subprocess.check_output("nvidia-smi --query-gpu=memory.used --format=csv", shell = True) + except: + raise RuntimeError("Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!") + output = re.findall(rb'([\d]{1,})[\s]{1,}M', output) + output = np.array([int(x.decode('utf-8'))/1024 for x in output]) + return output +pass +PRE_CHECK = check_nvidia() + + def create_boolean_mask(n = 4096, sliding_window = 2048): # Creates a boolean mask for attention mask = torch.ones(n, n, dtype = torch.bool) diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py index c7ae67e428..32f2c6b71c 100644 --- a/unsloth/models/llama.py +++ b/unsloth/models/llama.py @@ -83,7 +83,8 @@ def _fast_prepare_inputs_for_generation(self, input_ids, **kwargs,): if "past_key_values" in kwargs: input_ids = input_ids[:,[-1]] kwargs["attention_mask"] = kwargs["attention_mask"][:,[-1]] - kwargs["position_ids"] = kwargs["cache_position"] + if "cache_position" in kwargs: + kwargs["position_ids"] = kwargs["cache_position"] return { "input_ids" : input_ids, **kwargs, } pass @@ -1128,7 +1129,7 @@ class FastLlamaModel: f' "-____-" Free Apache license: http://github.com/unslothai/unsloth' print(statistics) model_patcher.pre_patch() - # get_statistics() + get_statistics() # For debugging - we use a download counter to see if environments are not breaking if dtype is None: dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16 @@ -1180,6 +1181,8 @@ class FastLlamaModel: # Add to kwargs kwargs["rope_scaling"] = rope_scaling pass + # We currently only support NVIDIA GPUs - AMD / Intel is a work in progress! + pre_check = check_nvidia() bnb_config = None if load_in_4bit: @@ -1206,6 +1209,8 @@ class FastLlamaModel: attn_implementation = "eager", **kwargs, ) + # We currently only support NVIDIA GPUs - AMD / Intel is a work in progress! + post_check = check_nvidia() # Counteract saved tokenizers tokenizer_name = model_name if tokenizer_name is None else tokenizer_name @@ -1235,14 +1240,12 @@ class FastLlamaModel: else: inner_training_loop = Trainer._original_training_loop except: - raise RuntimeError( - 'Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\ - 'enabling it will require much more work, so we have to prioritize. Please understand!\n'\ - 'We do have a separate beta version, which you can contact us about!\n'\ - 'Thank you for your understanding and we appreciate it immensely!' - ) + raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') pass + if ((post_check - post_check) >= 1).sum() > 1: + raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') + import transformers.trainer items_in_trainer = dir(transformers.trainer) good_items = [] @@ -1266,16 +1269,15 @@ class FastLlamaModel: f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\ f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}' logger.warning(debug_info) - import subprocess, re, gc - output = subprocess.check_output( - 'nvidia-smi --query-gpu=memory.used --format=csv', shell = True) - output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output) - output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output) - if output > 1: print( - '********************\\nUnsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\\ - 'enabling it will require much more work, so we have to prioritize. Please understand!\\n'\\ - '********************\\nWe do have a separate beta version, which you can contact us about!\\n'\\ - '********************\\nThank you for your understanding and we appreciate it immensely!') + import subprocess, re, gc, numpy as np + try: + a = subprocess.check_output('nvidia-smi --query-gpu=memory.used --format=csv', shell = True) + except: + raise RuntimeError('Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!') + a = re.findall(rb'([\\d]{1,})[\\s]{1,}M', a) + a = np.array([int(x.decode('utf-8'))/1024 for x in a]) + if ((a - PRE_CHECK) >= 1).sum() > 1: + raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') for _ in range(3): gc.collect() torch.cuda.empty_cache()""" @@ -1287,12 +1289,7 @@ class FastLlamaModel: debug_info = """n_total_devices = total_train_batch_size // \\ args.gradient_accumulation_steps // self._train_batch_size if n_total_devices > 1: - logger.warning_once( - '* Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so ' \\ - '* enabling it will require much more work, so we have to prioritize. Please understand!\\n' \\ - '* We do have a separate beta version, which you can contact us about!\\n'\\ - '* Thank you for your understanding and we appreciate it immensely!' - ) + logger.warning_once('Unsloth currently does not support multi GPU setups - but we are working on it!') debug_info =""" debug_info = debug_info.split('\n') debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]]) @@ -1317,12 +1314,7 @@ class FastLlamaModel: total_batches = bsz * ga * args.world_size n_total_devices = total_batches // ga // bsz if n_total_devices > 1: - logger.warning_once( - '* Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so ' \\ - '* enabling it will require much more work, so we have to prioritize. Please understand!\\n' \\ - '* We do have a separate beta version, which you can contact us about!\\n'\\ - '* Thank you for your understanding and we appreciate it immensely!' - ) + logger.warning_once('Unsloth currently does not support multi GPU setups - but we are working on it!') divisor = n_total_devices / 1 bsz = self._train_batch_size = max(int(bsz / divisor), 1) if total_batches // ga // bsz > 1: @@ -1346,12 +1338,7 @@ class FastLlamaModel: "False", ) if "n_total_devices >" not in inner_training_loop: - raise RuntimeError( - 'Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\ - 'enabling it will require much more work, so we have to prioritize. Please understand!\n'\ - 'We do have a separate beta version, which you can contact us about!\n'\ - 'Thank you for your understanding and we appreciate it immensely!' - ) + raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!') pass inner_training_loop = inner_training_loop.replace( "is_sagemaker_mp_enabled()", diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py index b0bc514f67..440a53c1f0 100644 --- a/unsloth/tokenizer_utils.py +++ b/unsloth/tokenizer_utils.py @@ -24,6 +24,7 @@ import itertools import collections import numpy as np import gc +import subprocess __all__ = [ "load_correct_tokenizer", @@ -907,6 +908,19 @@ def add_new_tokens( pass +def check_nvidia(): + # Unsloth doesn't work yet on AMD devices - we're working on it! + try: + output = subprocess.check_output("nvidia-smi --query-gpu=memory.used --format=csv", shell = True) + except: + raise RuntimeError("Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!") + output = re.findall(rb'([\d]{1,})[\s]{1,}M', output) + output = np.array([int(x.decode('utf-8'))/1024 for x in output]) + return output +pass +PRE_CHECK = check_nvidia() + + from inspect import getsource import trl.trainer.sft_trainer from trl.trainer.sft_trainer import * @@ -957,17 +971,15 @@ def patch_sft_trainer_tokenizer(): " 'Please do not edit specific areas of the Unsloth codebase or you will get CUDA segfaults.'\n"\ " )\n"\ "pass\n"\ - "n_devices = torch.cuda.device_count()\n"\ - "import subprocess, re\n"\ - "output = subprocess.check_output(\n"\ - " 'nvidia-smi --query-gpu=memory.used --format=csv', shell = True)\n"\ - "output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output)\n"\ - "output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output)\n"\ - "if output > 1: print(\n"\ - " '********************\\nUnsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\\\n"\ - " 'enabling it will require much more work, so we have to prioritize. Please understand!\\n'\\\n"\ - " '********************\\nWe do have a separate beta version, which you can contact us about!\\n'\\\n"\ - " '********************\\nThank you for your understanding and we appreciate it immensely!')\n"\ + "import subprocess, re, gc, numpy as np\n"\ + "try:\n"\ + " a = subprocess.check_output('nvidia-smi --query-gpu=memory.used --format=csv', shell = True)\n"\ + "except:\n"\ + " raise RuntimeError('Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!')\n"\ + "a = re.findall(rb'([\\d]{1,})[\\s]{1,}M', a)\n"\ + "a = np.array([int(x.decode('utf-8'))/1024 for x in a])\n"\ + "if ((a - PRE_CHECK) >= 1).sum() > 1:\n"\ + " raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!')\n"\ "for _ in range(3):\n"\ " gc.collect()\n"\ " torch.cuda.empty_cache()\n"\