original
This commit is contained in:
Daniel Han-Chen
parent
10d9d56434
commit
7372768d14
2 changed files with 216 additions and 212 deletions
|
|
@ -965,121 +965,123 @@ class FastLlamaModel:
|
||||||
|
|
||||||
# Patch Trainer
|
# Patch Trainer
|
||||||
from transformers.trainer import Trainer
|
from transformers.trainer import Trainer
|
||||||
if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
|
try:
|
||||||
try:
|
if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
|
||||||
inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
|
inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
|
||||||
except:
|
Trainer._original_training_loop = inner_training_loop
|
||||||
raise RuntimeError(
|
else:
|
||||||
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
inner_training_loop = Trainer._original_training_loop
|
||||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
except:
|
||||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
raise RuntimeError(
|
||||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
||||||
)
|
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
||||||
pass
|
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
||||||
|
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||||
import transformers.trainer
|
|
||||||
items_in_trainer = dir(transformers.trainer)
|
|
||||||
good_items = []
|
|
||||||
for item in items_in_trainer:
|
|
||||||
# TODO: Support Deepspeed
|
|
||||||
if item.startswith(("deepspeed", "xm", "met", "smp")): continue
|
|
||||||
if item in inner_training_loop: good_items.append(item)
|
|
||||||
pass
|
|
||||||
exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")", globals())
|
|
||||||
|
|
||||||
start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0]
|
|
||||||
end = inner_training_loop.find("\n\n", start)
|
|
||||||
original_debug = inner_training_loop[start:end]
|
|
||||||
spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:]
|
|
||||||
front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0)
|
|
||||||
|
|
||||||
debug_info = """debug_info = \\
|
|
||||||
f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs = {args.world_size}\\n"\\
|
|
||||||
f" \\\\\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\
|
|
||||||
f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\
|
|
||||||
f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\
|
|
||||||
f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}'
|
|
||||||
logger.warning_once(debug_info)"""
|
|
||||||
|
|
||||||
debug_info = debug_info.split('\n')
|
|
||||||
debug_info = "\n".join([debug_info[0]] + [spaces + x[16:] for x in debug_info[1:]])
|
|
||||||
inner_training_loop = inner_training_loop.replace(original_debug, debug_info)
|
|
||||||
|
|
||||||
debug_info = """n_total_devices = total_train_batch_size // \\
|
|
||||||
args.gradient_accumulation_steps // self._train_batch_size
|
|
||||||
if n_total_devices > 2:
|
|
||||||
logger.warning_once(
|
|
||||||
"Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
|
||||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n"
|
|
||||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
|
||||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
|
||||||
)
|
|
||||||
debug_info ="""
|
|
||||||
debug_info = debug_info.split('\n')
|
|
||||||
debug_info = "\n".join([debug_info[0]] + [spaces + x[16:] for x in debug_info[1:]])
|
|
||||||
inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1)
|
|
||||||
|
|
||||||
front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0)
|
|
||||||
inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags = re.MULTILINE)
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"train_dataloader = tpu_spmd_dataloader(train_dataloader)",
|
|
||||||
"raise RuntimeError('Unsloth: TPUs are not yet supported!')"
|
|
||||||
)
|
)
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"self.accelerator.free_memory()",
|
|
||||||
"self.accelerator.free_memory()\n" + \
|
|
||||||
front_spaces + "if self.is_deepspeed_enabled:"\
|
|
||||||
"raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
check_batches = """train_dataloader = self.get_train_dataloader()
|
|
||||||
ga = args.gradient_accumulation_steps
|
|
||||||
bsz = self._train_batch_size
|
|
||||||
total_batches = bsz * ga * args.world_size
|
|
||||||
n_total_devices = total_batches // ga // bsz
|
|
||||||
if n_total_devices > 2:
|
|
||||||
logger.warning_once(
|
|
||||||
"Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n"
|
|
||||||
"The OSS currently works on 4 GPUs - we're a 2 person team, so please help fund\\n"
|
|
||||||
"our development costs by supporting us through Ko-fi or buying a license! Thanks!",
|
|
||||||
)
|
|
||||||
divisor = n_total_devices / 2
|
|
||||||
bsz = self._train_batch_size = max(int(bsz / divisor), 1)
|
|
||||||
if total_batches // ga // bsz > 2:
|
|
||||||
divisor = n_total_devices / 2
|
|
||||||
ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)"""
|
|
||||||
check_batches = check_batches.split('\n')
|
|
||||||
check_batches = "\n".join([check_batches[0]] + [front_spaces + x[16:] for x in check_batches[1:]])
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"train_dataloader = self.get_train_dataloader()",
|
|
||||||
check_batches, 1,
|
|
||||||
)
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"_inner_training_loop",
|
|
||||||
"_fast_inner_training_loop", 1,
|
|
||||||
)
|
|
||||||
exec(inner_training_loop, globals())
|
|
||||||
|
|
||||||
Trainer._inner_training_loop = _fast_inner_training_loop
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"is_torch_tpu_available()",
|
|
||||||
"False",
|
|
||||||
)
|
|
||||||
if "n_total_devices >" not in inner_training_loop:
|
|
||||||
raise RuntimeError(
|
|
||||||
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
|
||||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
|
||||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
|
||||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
|
||||||
)
|
|
||||||
pass
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"is_sagemaker_mp_enabled()",
|
|
||||||
"False",
|
|
||||||
)
|
|
||||||
Trainer._inner_training_loop = _fast_inner_training_loop
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
import transformers.trainer
|
||||||
|
items_in_trainer = dir(transformers.trainer)
|
||||||
|
good_items = []
|
||||||
|
for item in items_in_trainer:
|
||||||
|
# TODO: Support Deepspeed
|
||||||
|
if item.startswith(("deepspeed", "xm", "met", "smp")): continue
|
||||||
|
if item in inner_training_loop: good_items.append(item)
|
||||||
|
pass
|
||||||
|
exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")", globals())
|
||||||
|
|
||||||
|
start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0]
|
||||||
|
end = inner_training_loop.find("\n\n", start)
|
||||||
|
original_debug = inner_training_loop[start:end]
|
||||||
|
spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:]
|
||||||
|
front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0)
|
||||||
|
|
||||||
|
debug_info = """debug_info = \\
|
||||||
|
f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs = {args.world_size}\\n"\\
|
||||||
|
f" \\\\\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\
|
||||||
|
f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\
|
||||||
|
f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\
|
||||||
|
f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}'
|
||||||
|
logger.warning_once(debug_info)"""
|
||||||
|
|
||||||
|
debug_info = debug_info.split('\n')
|
||||||
|
debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]])
|
||||||
|
inner_training_loop = inner_training_loop.replace(original_debug, debug_info)
|
||||||
|
|
||||||
|
debug_info = """n_total_devices = total_train_batch_size // \\
|
||||||
|
args.gradient_accumulation_steps // self._train_batch_size
|
||||||
|
if n_total_devices > 2:
|
||||||
|
logger.warning_once(
|
||||||
|
"Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||||
|
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n"
|
||||||
|
"We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||||
|
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||||
|
)
|
||||||
|
debug_info ="""
|
||||||
|
debug_info = debug_info.split('\n')
|
||||||
|
debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]])
|
||||||
|
inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1)
|
||||||
|
|
||||||
|
front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0)
|
||||||
|
inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags = re.MULTILINE)
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"train_dataloader = tpu_spmd_dataloader(train_dataloader)",
|
||||||
|
"raise RuntimeError('Unsloth: TPUs are not yet supported!')"
|
||||||
|
)
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"self.accelerator.free_memory()",
|
||||||
|
"self.accelerator.free_memory()\n" + \
|
||||||
|
front_spaces + "if self.is_deepspeed_enabled:"\
|
||||||
|
"raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1,
|
||||||
|
)
|
||||||
|
|
||||||
|
check_batches = """train_dataloader = self.get_train_dataloader()
|
||||||
|
ga = args.gradient_accumulation_steps
|
||||||
|
bsz = self._train_batch_size
|
||||||
|
total_batches = bsz * ga * args.world_size
|
||||||
|
n_total_devices = total_batches // ga // bsz
|
||||||
|
if n_total_devices > 2:
|
||||||
|
logger.warning_once(
|
||||||
|
"Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n"
|
||||||
|
"The OSS currently works on 4 GPUs - we're a 2 person team, so please help fund\\n"
|
||||||
|
"our development costs by supporting us through Ko-fi or buying a license! Thanks!",
|
||||||
|
)
|
||||||
|
divisor = n_total_devices / 2
|
||||||
|
bsz = self._train_batch_size = max(int(bsz / divisor), 1)
|
||||||
|
if total_batches // ga // bsz > 2:
|
||||||
|
divisor = n_total_devices / 2
|
||||||
|
ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)"""
|
||||||
|
check_batches = check_batches.split('\n')
|
||||||
|
check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]])
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"train_dataloader = self.get_train_dataloader()",
|
||||||
|
check_batches, 1,
|
||||||
|
)
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"_inner_training_loop",
|
||||||
|
"_fast_inner_training_loop", 1,
|
||||||
|
)
|
||||||
|
exec(inner_training_loop, globals())
|
||||||
|
|
||||||
|
Trainer._inner_training_loop = _fast_inner_training_loop
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"is_torch_tpu_available()",
|
||||||
|
"False",
|
||||||
|
)
|
||||||
|
if "n_total_devices >" not in inner_training_loop:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
||||||
|
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
||||||
|
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
||||||
|
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||||
|
)
|
||||||
|
pass
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"is_sagemaker_mp_enabled()",
|
||||||
|
"False",
|
||||||
|
)
|
||||||
|
Trainer._inner_training_loop = _fast_inner_training_loop
|
||||||
|
|
||||||
# Save max_seq_length
|
# Save max_seq_length
|
||||||
model.max_seq_length = max_position_embeddings
|
model.max_seq_length = max_position_embeddings
|
||||||
internal_model = model
|
internal_model = model
|
||||||
|
|
|
||||||
|
|
@ -385,121 +385,123 @@ class FastMistralModel(FastLlamaModel):
|
||||||
|
|
||||||
# Patch Trainer
|
# Patch Trainer
|
||||||
from transformers.trainer import Trainer
|
from transformers.trainer import Trainer
|
||||||
if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
|
try:
|
||||||
try:
|
if Trainer._inner_training_loop.__name__ != "_fast_inner_training_loop":
|
||||||
inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
|
inner_training_loop = inspect.getsource(Trainer._inner_training_loop)
|
||||||
except:
|
Trainer._original_training_loop = inner_training_loop
|
||||||
raise RuntimeError(
|
else:
|
||||||
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
inner_training_loop = Trainer._original_training_loop
|
||||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
except:
|
||||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
raise RuntimeError(
|
||||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
||||||
)
|
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
||||||
pass
|
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
||||||
|
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||||
import transformers.trainer
|
|
||||||
items_in_trainer = dir(transformers.trainer)
|
|
||||||
good_items = []
|
|
||||||
for item in items_in_trainer:
|
|
||||||
# TODO: Support Deepspeed
|
|
||||||
if item.startswith(("deepspeed", "xm", "met", "smp")): continue
|
|
||||||
if item in inner_training_loop: good_items.append(item)
|
|
||||||
pass
|
|
||||||
exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")", globals())
|
|
||||||
|
|
||||||
start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0]
|
|
||||||
end = inner_training_loop.find("\n\n", start)
|
|
||||||
original_debug = inner_training_loop[start:end]
|
|
||||||
spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:]
|
|
||||||
front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0)
|
|
||||||
|
|
||||||
debug_info = """debug_info = \\
|
|
||||||
f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs = {args.world_size}\\n"\\
|
|
||||||
f" \\\\\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\
|
|
||||||
f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\
|
|
||||||
f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\
|
|
||||||
f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}'
|
|
||||||
logger.warning_once(debug_info)"""
|
|
||||||
|
|
||||||
debug_info = debug_info.split('\n')
|
|
||||||
debug_info = "\n".join([debug_info[0]] + [spaces + x[16:] for x in debug_info[1:]])
|
|
||||||
inner_training_loop = inner_training_loop.replace(original_debug, debug_info)
|
|
||||||
|
|
||||||
debug_info = """n_total_devices = total_train_batch_size // \\
|
|
||||||
args.gradient_accumulation_steps // self._train_batch_size
|
|
||||||
if n_total_devices > 2:
|
|
||||||
logger.warning_once(
|
|
||||||
"Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
|
||||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n"
|
|
||||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
|
||||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
|
||||||
)
|
|
||||||
debug_info ="""
|
|
||||||
debug_info = debug_info.split('\n')
|
|
||||||
debug_info = "\n".join([debug_info[0]] + [spaces + x[16:] for x in debug_info[1:]])
|
|
||||||
inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1)
|
|
||||||
|
|
||||||
front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0)
|
|
||||||
inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags = re.MULTILINE)
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"train_dataloader = tpu_spmd_dataloader(train_dataloader)",
|
|
||||||
"raise RuntimeError('Unsloth: TPUs are not yet supported!')"
|
|
||||||
)
|
)
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"self.accelerator.free_memory()",
|
|
||||||
"self.accelerator.free_memory()\n" + \
|
|
||||||
front_spaces + "if self.is_deepspeed_enabled:"\
|
|
||||||
"raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
check_batches = """train_dataloader = self.get_train_dataloader()
|
|
||||||
ga = args.gradient_accumulation_steps
|
|
||||||
bsz = self._train_batch_size
|
|
||||||
total_batches = bsz * ga * args.world_size
|
|
||||||
n_total_devices = total_batches // ga // bsz
|
|
||||||
if n_total_devices > 2:
|
|
||||||
logger.warning_once(
|
|
||||||
"Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n"
|
|
||||||
"The OSS currently works on 4 GPUs - we're a 2 person team, so please help fund\\n"
|
|
||||||
"our development costs by supporting us through Ko-fi or buying a license! Thanks!",
|
|
||||||
)
|
|
||||||
divisor = n_total_devices / 2
|
|
||||||
bsz = self._train_batch_size = max(int(bsz / divisor), 1)
|
|
||||||
if total_batches // ga // bsz > 2:
|
|
||||||
divisor = n_total_devices / 2
|
|
||||||
ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)"""
|
|
||||||
check_batches = check_batches.split('\n')
|
|
||||||
check_batches = "\n".join([check_batches[0]] + [front_spaces + x[16:] for x in check_batches[1:]])
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"train_dataloader = self.get_train_dataloader()",
|
|
||||||
check_batches, 1,
|
|
||||||
)
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"_inner_training_loop",
|
|
||||||
"_fast_inner_training_loop", 1,
|
|
||||||
)
|
|
||||||
exec(inner_training_loop, globals())
|
|
||||||
|
|
||||||
Trainer._inner_training_loop = _fast_inner_training_loop
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"is_torch_tpu_available()",
|
|
||||||
"False",
|
|
||||||
)
|
|
||||||
if "n_total_devices >" not in inner_training_loop:
|
|
||||||
raise RuntimeError(
|
|
||||||
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
|
||||||
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
|
||||||
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
|
||||||
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
|
||||||
)
|
|
||||||
pass
|
|
||||||
inner_training_loop = inner_training_loop.replace(
|
|
||||||
"is_sagemaker_mp_enabled()",
|
|
||||||
"False",
|
|
||||||
)
|
|
||||||
Trainer._inner_training_loop = _fast_inner_training_loop
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
import transformers.trainer
|
||||||
|
items_in_trainer = dir(transformers.trainer)
|
||||||
|
good_items = []
|
||||||
|
for item in items_in_trainer:
|
||||||
|
# TODO: Support Deepspeed
|
||||||
|
if item.startswith(("deepspeed", "xm", "met", "smp")): continue
|
||||||
|
if item in inner_training_loop: good_items.append(item)
|
||||||
|
pass
|
||||||
|
exec("from transformers.trainer import (" + ", ".join(x for x in good_items) + ")", globals())
|
||||||
|
|
||||||
|
start = re.search('logger\.info\([\"\'].+?Running training', inner_training_loop).span(0)[0]
|
||||||
|
end = inner_training_loop.find("\n\n", start)
|
||||||
|
original_debug = inner_training_loop[start:end]
|
||||||
|
spaces = re.search('\n([\s\t]{1,})', original_debug).group(0)[1:]
|
||||||
|
front_spaces = re.match('([\s\t]{1,})', inner_training_loop).group(0)
|
||||||
|
|
||||||
|
debug_info = """debug_info = \\
|
||||||
|
f"==((====))== Unsloth - 2x faster free finetuning | Num GPUs = {args.world_size}\\n"\\
|
||||||
|
f" \\\\\\ /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,}\\n"\\
|
||||||
|
f"O^O/ \\_/ \\ Batch size per device = {self._train_batch_size:,} | Gradient Accumulation steps = {args.gradient_accumulation_steps}\\n"\\
|
||||||
|
f"\\ / Total batch size = {total_train_batch_size:,} | Total steps = {max_steps:,}\\n"\\
|
||||||
|
f' "-____-" Number of trainable parameters = {get_model_param_count(model, trainable_only=True):,}'
|
||||||
|
logger.warning_once(debug_info)"""
|
||||||
|
|
||||||
|
debug_info = debug_info.split('\n')
|
||||||
|
debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]])
|
||||||
|
inner_training_loop = inner_training_loop.replace(original_debug, debug_info)
|
||||||
|
|
||||||
|
debug_info = """n_total_devices = total_train_batch_size // \\
|
||||||
|
args.gradient_accumulation_steps // self._train_batch_size
|
||||||
|
if n_total_devices > 2:
|
||||||
|
logger.warning_once(
|
||||||
|
"Our OSS was designed for people with few GPU resources to level the playing field.\\n"
|
||||||
|
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\\n"
|
||||||
|
"We're a 2 person team, so we still have to fund our development costs - thanks!\\n"
|
||||||
|
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||||
|
)
|
||||||
|
debug_info ="""
|
||||||
|
debug_info = debug_info.split('\n')
|
||||||
|
debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]])
|
||||||
|
inner_training_loop = inner_training_loop.replace("debug_info =", debug_info, 1)
|
||||||
|
|
||||||
|
front_spaces = re.match(r"[\t\s]{1,}", inner_training_loop).group(0)
|
||||||
|
inner_training_loop = re.sub(r"^" + front_spaces, "", inner_training_loop, flags = re.MULTILINE)
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"train_dataloader = tpu_spmd_dataloader(train_dataloader)",
|
||||||
|
"raise RuntimeError('Unsloth: TPUs are not yet supported!')"
|
||||||
|
)
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"self.accelerator.free_memory()",
|
||||||
|
"self.accelerator.free_memory()\n" + \
|
||||||
|
front_spaces + "if self.is_deepspeed_enabled:"\
|
||||||
|
"raise RuntimeError('Unsloth: Deepspeed is not yet supported!')\n", 1,
|
||||||
|
)
|
||||||
|
|
||||||
|
check_batches = """train_dataloader = self.get_train_dataloader()
|
||||||
|
ga = args.gradient_accumulation_steps
|
||||||
|
bsz = self._train_batch_size
|
||||||
|
total_batches = bsz * ga * args.world_size
|
||||||
|
n_total_devices = total_batches // ga // bsz
|
||||||
|
if n_total_devices > 2:
|
||||||
|
logger.warning_once(
|
||||||
|
"Please consider a commercial license - Unsloth was designed for the GPU Poor.\\n"
|
||||||
|
"The OSS currently works on 4 GPUs - we're a 2 person team, so please help fund\\n"
|
||||||
|
"our development costs by supporting us through Ko-fi or buying a license! Thanks!",
|
||||||
|
)
|
||||||
|
divisor = n_total_devices / 2
|
||||||
|
bsz = self._train_batch_size = max(int(bsz / divisor), 1)
|
||||||
|
if total_batches // ga // bsz > 2:
|
||||||
|
divisor = n_total_devices / 2
|
||||||
|
ga = args.gradient_accumulation_steps = max(int(ga / divisor), 1)"""
|
||||||
|
check_batches = check_batches.split('\n')
|
||||||
|
check_batches = "\n".join([check_batches[0]] + [front_spaces + x[8:] for x in check_batches[1:]])
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"train_dataloader = self.get_train_dataloader()",
|
||||||
|
check_batches, 1,
|
||||||
|
)
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"_inner_training_loop",
|
||||||
|
"_fast_inner_training_loop", 1,
|
||||||
|
)
|
||||||
|
exec(inner_training_loop, globals())
|
||||||
|
|
||||||
|
Trainer._inner_training_loop = _fast_inner_training_loop
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"is_torch_tpu_available()",
|
||||||
|
"False",
|
||||||
|
)
|
||||||
|
if "n_total_devices >" not in inner_training_loop:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Our OSS was designed for people with few GPU resources to level the playing field.\n"
|
||||||
|
"The OSS Apache 2 license only supports four GPUs - please obtain a commercial license from our website.\n"
|
||||||
|
"We're a 2 person team, so we still have to fund our development costs - thanks!\n"
|
||||||
|
"If you don't, please consider at least sponsoring us through Ko-fi! Appreciate it!",
|
||||||
|
)
|
||||||
|
pass
|
||||||
|
inner_training_loop = inner_training_loop.replace(
|
||||||
|
"is_sagemaker_mp_enabled()",
|
||||||
|
"False",
|
||||||
|
)
|
||||||
|
Trainer._inner_training_loop = _fast_inner_training_loop
|
||||||
|
|
||||||
# Save max_seq_length
|
# Save max_seq_length
|
||||||
max_position_embeddings = max(max_seq_length, model.config.max_position_embeddings)
|
max_position_embeddings = max(max_seq_length, model.config.max_position_embeddings)
|
||||||
model.max_seq_length = max_position_embeddings
|
model.max_seq_length = max_position_embeddings
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue