ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update llama.py

Browse source
This commit is contained in:
Daniel Han 2025-03-14 01:36:12 -07:00
commit 538a5df0de
1 changed files with 7 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

13
unsloth/models/llama.py
View file

@ -1883,17 +1883,18 @@ class FastLlamaModel:
f" {chr(92)}{chr(92)} /| Num examples = {num_examples:,} | Num Epochs = {num_train_epochs:,} | Total steps = {max_steps:,}\\n"\\
f"O^O/ {chr(92)}_/ {chr(92)} Batch size per device = {self._train_batch_size:,} | Gradient accumulation steps = {args.gradient_accumulation_steps}\\n"\\
f"{chr(92)} / Data Parallel GPUs = {args.world_size} | Total batch size ({self._train_batch_size} x {args.gradient_accumulation_steps} x {args.world_size}) = {total_train_batch_size:,}\\n"\\
f' "-____-" Trainable parameters = {P__(model, trainable_only=True):,}/{P__(model)*multiplier__:,} ({P__(model, trainable_only=True)/(P__(model)*multiplier__)*100:.2f}% trained)'
f' "-____-" Trainable parameters = {!!(model, trainable_only=True):,}/{!!(model)*($$):,} ({!!(model, trainable_only=True)/(!!(model)*($$))*100:.2f}% trained)'
logger.warning(debug_info)
import gc
for _ in range(3):
gc.collect()
torch.cuda.empty_cache()"""
multiplier = \
"(4.5 if getattr(model.config, 'quantization_config', \\{'load_in_4bit' : False\\})['load_in_4bit'] else "\
"(8.0 if getattr(model.config, 'quantization_config', \\{'load_in_8bit' : False\\})['load_in_8bit'] else 1.0)"
debug_info = debug_info.replace("multiplier__", multiplier)
debug_info = debug_info.replace("P__", "get_model_param_count")
debug_info = debug_info.replace("!!", "get_model_param_count")
debug_info = debug_info.replace(
"$$",
"(4.5 if getattr(model, 'quantization_config', '\\{'load_in_4bit':False\\}')['load_in_4bit'] else "\
"(8.0 if getattr(model, 'quantization_config', '\\{'load_in_8bit':False\\}')['load_in_8bit'] else 1.0))"
)
debug_info = debug_info.split('\n')
debug_info = "\n".join([debug_info[0]] + [spaces + x[8:] for x in debug_info[1:]])