Debugging (#739)
* Update gemma2.py * Update llama.py * Update llama.py * Update gemma2.py * init * Update gemma2.py * Update gemma2.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update _utils.py * Update gemma2.py * Update gemma2.py * Update gemma2.py * All RoPE Scaling support * cleanup * Update llama.py * Update llama.py * Update _utils.py * Update _utils.py * exec * exec * Attention_Module * attention_module * imports * exec * Update llama.py * Update llama.py * boolean mask * revert masking * Update llama.py * Update save.py * Update llama.py * Update gemma2.py * Update gemma2.py * Update gemma2.py * Update utils.py * retry * Update gemma2.py * Update gemma2.py * Update gemma2.py * Update _utils.py * Update _utils.py * Update gemma2.py * Update chat_templates.py * Gemma 2 Ollama support * Update llama.py * Update llama.py * error handling * Update _utils.py * Update _utils.py * Stats for debugging * Update _utils.py * Update _utils.py * Debugging * Update tokenizer_utils.py * Update _utils.py
This commit is contained in:
Daniel Han
GitHub
parent
86c5675a67
commit
75df21a314
4 changed files with 76 additions and 61 deletions
|
|
@ -39,10 +39,7 @@ if "CUDA_VISIBLE_DEVICES" in os.environ:
|
|||
first_id = devices.split(",")[0]
|
||||
warnings.warn(
|
||||
f"Unsloth: 'CUDA_VISIBLE_DEVICES' is currently {devices} \n"\
|
||||
"Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so "\
|
||||
"enabling it will require much more work, so we have to prioritize. Please understand!"\
|
||||
"We do have a beta version, which you can contact us about!\n"\
|
||||
"Thank you for your understanding and we appreciate it immensely!\n\n"\
|
||||
"Unsloth currently does not support multi GPU setups - but we are working on it!\n"\
|
||||
"Multiple CUDA devices detected but we require a single device.\n"\
|
||||
f"We will override CUDA_VISIBLE_DEVICES to first device: {first_id}."
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ __all__ = [
|
|||
"xformers_version",
|
||||
"__version__",
|
||||
"HAS_FLASH_ATTENTION",
|
||||
"PRE_CHECK",
|
||||
"platform_system",
|
||||
"patch_tokenizer",
|
||||
"get_statistics",
|
||||
|
|
@ -32,30 +33,27 @@ __all__ = [
|
|||
"unsloth_offloaded_gradient_checkpoint",
|
||||
"torch_compile_options",
|
||||
"patch_linear_scaling",
|
||||
"check_nvidia",
|
||||
"create_boolean_mask",
|
||||
]
|
||||
|
||||
import torch
|
||||
from typing import Union, Optional, List, Any, Callable, Tuple
|
||||
import warnings
|
||||
from platform import system as platform_system
|
||||
platform_system = platform_system()
|
||||
import math
|
||||
import numpy as np
|
||||
import os
|
||||
import psutil
|
||||
import inspect
|
||||
import re
|
||||
import warnings, subprocess, re, inspect, psutil, os, math
|
||||
|
||||
# =============================================
|
||||
# Disable some warnings which can get annoying
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "torch")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "huggingface_hub")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "huggingface_hub")
|
||||
warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "subprocess")
|
||||
warnings.filterwarnings(action = "ignore", category = UserWarning, module = "transformers")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "accelerate")
|
||||
warnings.filterwarnings(action = "ignore", category = FutureWarning, module = "huggingface_hub")
|
||||
warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "multiprocessing")
|
||||
warnings.filterwarnings(action = "ignore", category = RuntimeWarning, module = "multiprocess")
|
||||
|
||||
# Stop "Special tokens have been added in the vocabulary, ..."
|
||||
import logging
|
||||
|
|
@ -74,7 +72,10 @@ for model_name in model_architectures:
|
|||
config_filename = f"{model_name.title()}Config"
|
||||
exec(f"from {config_filepath} import {config_filename}", globals())
|
||||
|
||||
config = inspect.getsource(eval(config_filename))
|
||||
try:
|
||||
config = inspect.getsource(eval(config_filename))
|
||||
except:
|
||||
continue
|
||||
if "rope_scaling" in config: continue
|
||||
config = re.sub(
|
||||
r"(\*\*kwargs)[\s]{0,}\,[\s]{0,}\)[\s]{0,}\:",
|
||||
|
|
@ -345,7 +346,6 @@ def get_statistics():
|
|||
# We simply download a README.md file from HF - all data is made public.
|
||||
# This is simply so we can check if some envs are broken or not.
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
from huggingface_hub.utils import disable_progress_bars, enable_progress_bars, are_progress_bars_disabled
|
||||
import psutil
|
||||
n_cpus = psutil.cpu_count(logical = False)
|
||||
|
|
@ -367,7 +367,13 @@ def get_statistics():
|
|||
disable_progress_bars()
|
||||
disabled = True
|
||||
pass
|
||||
hf_hub_download(f"unslothai/statistics-{statistics}", "README.md", force_download = True)
|
||||
|
||||
from transformers import AutoModelForCausalLM
|
||||
stats_model = AutoModelForCausalLM.from_pretrained(
|
||||
f"unslothai/statistics-{statistics}",
|
||||
force_download = True,
|
||||
)
|
||||
del stats_model
|
||||
if disabled:
|
||||
enable_progress_bars()
|
||||
pass
|
||||
|
|
@ -659,6 +665,19 @@ def patch_linear_scaling(
|
|||
pass
|
||||
|
||||
|
||||
def check_nvidia():
|
||||
# Unsloth doesn't work yet on AMD devices - we're working on it!
|
||||
try:
|
||||
output = subprocess.check_output("nvidia-smi --query-gpu=memory.used --format=csv", shell = True)
|
||||
except:
|
||||
raise RuntimeError("Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!")
|
||||
output = re.findall(rb'([\d]{1,})[\s]{1,}M', output)
|
||||
output = np.array([int(x.decode('utf-8'))/1024 for x in output])
|
||||
return output
|
||||
pass
|
||||
PRE_CHECK = check_nvidia()
|
||||
|
||||
|
||||
def create_boolean_mask(n = 4096, sliding_window = 2048):
|
||||
# Creates a boolean mask for attention
|
||||
mask = torch.ones(n, n, dtype = torch.bool)
|
||||
|
|
|
|||
|
|
@ -83,7 +83,8 @@ def _fast_prepare_inputs_for_generation(self, input_ids, **kwargs,):
|
|||
if "past_key_values" in kwargs:
|
||||
input_ids = input_ids[:,[-1]]
|
||||
kwargs["attention_mask"] = kwargs["attention_mask"][:,[-1]]
|
||||
kwargs["position_ids"] = kwargs["cache_position"]
|
||||
if "cache_position" in kwargs:
|
||||
kwargs["position_ids"] = kwargs["cache_position"]
|
||||
return { "input_ids" : input_ids, **kwargs, }
|
||||
pass
|
||||
|
||||
|
|
@ -1128,7 +1129,7 @@ class FastLlamaModel:
|
|||
f' "-____-" Free Apache license: http://github.com/unslothai/unsloth'
|
||||
print(statistics)
|
||||
model_patcher.pre_patch()
|
||||
# get_statistics()
|
||||
get_statistics() # For debugging - we use a download counter to see if environments are not breaking
|
||||
|
||||
if dtype is None:
|
||||
dtype = torch.float16 if not SUPPORTS_BFLOAT16 else torch.bfloat16
|
||||
|
|
@ -1180,6 +1181,8 @@ class FastLlamaModel:
|
|||
# Add to kwargs
|
||||
kwargs["rope_scaling"] = rope_scaling
|
||||
pass
|
||||
# We currently only support NVIDIA GPUs - AMD / Intel is a work in progress!
|
||||
pre_check = check_nvidia()
|
||||
|
||||
bnb_config = None
|
||||
if load_in_4bit:
|
||||
|
|
@ -1206,6 +1209,8 @@ class FastLlamaModel:
|
|||
attn_implementation = "eager",
|
||||
**kwargs,
|
||||
)
|
||||
# We currently only support NVIDIA GPUs - AMD / Intel is a work in progress!
|
||||
post_check = check_nvidia()
|
||||
|
||||
# Counteract saved tokenizers
|
||||
tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
|
||||
|
|
@ -1235,14 +1240,12 @@ class FastLlamaModel:
|
|||
else:
|
||||
inner_training_loop = Trainer._original_training_loop
|
||||
except:
|
||||
raise RuntimeError(
|
||||
'Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\
|
||||
'enabling it will require much more work, so we have to prioritize. Please understand!\n'\
|
||||
'We do have a separate beta version, which you can contact us about!\n'\
|
||||
'Thank you for your understanding and we appreciate it immensely!'
|
||||
)
|
||||
raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!')
|
||||
pass
|
||||
|
||||
if ((post_check - post_check) >= 1).sum() > 1:
|
||||
raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!')
|
||||
|
||||
import transformers.trainer
|
||||
items_in_trainer = dir(transformers.trainer)
|
||||
good_items = []
|
||||
|
|
@ -1266,16 +1269,15 @@ class FastLlamaModel:
|
|||
f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\
|
||||
f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}'
|
||||
logger.warning(debug_info)
|
||||
import subprocess, re, gc
|
||||
output = subprocess.check_output(
|
||||
'nvidia-smi --query-gpu=memory.used --format=csv', shell = True)
|
||||
output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output)
|
||||
output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output)
|
||||
if output > 1: print(
|
||||
'********************\\nUnsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\\
|
||||
'enabling it will require much more work, so we have to prioritize. Please understand!\\n'\\
|
||||
'********************\\nWe do have a separate beta version, which you can contact us about!\\n'\\
|
||||
'********************\\nThank you for your understanding and we appreciate it immensely!')
|
||||
import subprocess, re, gc, numpy as np
|
||||
try:
|
||||
a = subprocess.check_output('nvidia-smi --query-gpu=memory.used --format=csv', shell = True)
|
||||
except:
|
||||
raise RuntimeError('Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!')
|
||||
a = re.findall(rb'([\\d]{1,})[\\s]{1,}M', a)
|
||||
a = np.array([int(x.decode('utf-8'))/1024 for x in a])
|
||||
if ((a - PRE_CHECK) >= 1).sum() > 1:
|
||||
raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!')
|
||||
for _ in range(3):
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()"""
|
||||
|
|
@ -1287,12 +1289,7 @@ class FastLlamaModel:
|
|||
debug_info = """n_total_devices = total_train_batch_size // \\
|
||||
args.gradient_accumulation_steps // self._train_batch_size
|
||||
if n_total_devices > 1:
|
||||
logger.warning_once(
|
||||
'* Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so ' \\
|
||||
'* enabling it will require much more work, so we have to prioritize. Please understand!\\n' \\
|
||||
'* We do have a separate beta version, which you can contact us about!\\n'\\
|
||||
'* Thank you for your understanding and we appreciate it immensely!'
|
||||
)
|
||||
logger.warning_once('Unsloth currently does not support multi GPU setups - but we are working on it!')
|
||||
debug_info ="""
|
||||
debug_info = debug_info.split('\n')
|
||||
debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]])
|
||||
|
|
@ -1317,12 +1314,7 @@ class FastLlamaModel:
|
|||
total_batches = bsz * ga * args.world_size
|
||||
n_total_devices = total_batches // ga // bsz
|
||||
if n_total_devices > 1:
|
||||
logger.warning_once(
|
||||
'* Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so ' \\
|
||||
'* enabling it will require much more work, so we have to prioritize. Please understand!\\n' \\
|
||||
'* We do have a separate beta version, which you can contact us about!\\n'\\
|
||||
'* Thank you for your understanding and we appreciate it immensely!'
|
||||
)
|
||||
logger.warning_once('Unsloth currently does not support multi GPU setups - but we are working on it!')
|
||||
divisor = n_total_devices / 1
|
||||
bsz = self._train_batch_size = max(int(bsz / divisor), 1)
|
||||
if total_batches // ga // bsz > 1:
|
||||
|
|
@ -1346,12 +1338,7 @@ class FastLlamaModel:
|
|||
"False",
|
||||
)
|
||||
if "n_total_devices >" not in inner_training_loop:
|
||||
raise RuntimeError(
|
||||
'Unsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\
|
||||
'enabling it will require much more work, so we have to prioritize. Please understand!\n'\
|
||||
'We do have a separate beta version, which you can contact us about!\n'\
|
||||
'Thank you for your understanding and we appreciate it immensely!'
|
||||
)
|
||||
raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!')
|
||||
pass
|
||||
inner_training_loop = inner_training_loop.replace(
|
||||
"is_sagemaker_mp_enabled()",
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import itertools
|
|||
import collections
|
||||
import numpy as np
|
||||
import gc
|
||||
import subprocess
|
||||
|
||||
__all__ = [
|
||||
"load_correct_tokenizer",
|
||||
|
|
@ -907,6 +908,19 @@ def add_new_tokens(
|
|||
pass
|
||||
|
||||
|
||||
def check_nvidia():
|
||||
# Unsloth doesn't work yet on AMD devices - we're working on it!
|
||||
try:
|
||||
output = subprocess.check_output("nvidia-smi --query-gpu=memory.used --format=csv", shell = True)
|
||||
except:
|
||||
raise RuntimeError("Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!")
|
||||
output = re.findall(rb'([\d]{1,})[\s]{1,}M', output)
|
||||
output = np.array([int(x.decode('utf-8'))/1024 for x in output])
|
||||
return output
|
||||
pass
|
||||
PRE_CHECK = check_nvidia()
|
||||
|
||||
|
||||
from inspect import getsource
|
||||
import trl.trainer.sft_trainer
|
||||
from trl.trainer.sft_trainer import *
|
||||
|
|
@ -957,17 +971,15 @@ def patch_sft_trainer_tokenizer():
|
|||
" 'Please do not edit specific areas of the Unsloth codebase or you will get CUDA segfaults.'\n"\
|
||||
" )\n"\
|
||||
"pass\n"\
|
||||
"n_devices = torch.cuda.device_count()\n"\
|
||||
"import subprocess, re\n"\
|
||||
"output = subprocess.check_output(\n"\
|
||||
" 'nvidia-smi --query-gpu=memory.used --format=csv', shell = True)\n"\
|
||||
"output = re.findall(rb'([\\d]{1,})[\\s]{1,}M', output)\n"\
|
||||
"output = sum(int(x.decode('utf-8'))/1024 > 4 for x in output)\n"\
|
||||
"if output > 1: print(\n"\
|
||||
" '********************\\nUnsloth currently does not work on multi GPU setups - sadly we are a 2 brother team so '\\\n"\
|
||||
" 'enabling it will require much more work, so we have to prioritize. Please understand!\\n'\\\n"\
|
||||
" '********************\\nWe do have a separate beta version, which you can contact us about!\\n'\\\n"\
|
||||
" '********************\\nThank you for your understanding and we appreciate it immensely!')\n"\
|
||||
"import subprocess, re, gc, numpy as np\n"\
|
||||
"try:\n"\
|
||||
" a = subprocess.check_output('nvidia-smi --query-gpu=memory.used --format=csv', shell = True)\n"\
|
||||
"except:\n"\
|
||||
" raise RuntimeError('Unsloth: We do not support AMD / Intel machines yet - it is a work in progress!')\n"\
|
||||
"a = re.findall(rb'([\\d]{1,})[\\s]{1,}M', a)\n"\
|
||||
"a = np.array([int(x.decode('utf-8'))/1024 for x in a])\n"\
|
||||
"if ((a - PRE_CHECK) >= 1).sum() > 1:\n"\
|
||||
" raise RuntimeError('Unsloth currently does not support multi GPU setups - but we are working on it!')\n"\
|
||||
"for _ in range(3):\n"\
|
||||
" gc.collect()\n"\
|
||||
" torch.cuda.empty_cache()\n"\
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue