ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scale dataset num_proc dynamically to cpu_count//3 instead of hardcap 8

Browse source
This commit is contained in:
Roland Tannous 2026-02-18 08:38:53 +00:00
commit d69431fa57
2 changed files with 5 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

4
studio/backend/utils/datasets/chat_templates.py
View file

@ -285,7 +285,7 @@ def apply_chat_template_to_dataset(
if not isinstance(dataset, IterableDataset):
from multiprocessing import cpu_count
if num_proc is None or type(num_proc) is not int:
num_proc = min(cpu_count(), 8)
num_proc = max(1, cpu_count() // 3)
dataset_map_kwargs['num_proc'] = num_proc
dataset_map_kwargs['desc'] = "Applying template to Alpaca format"
@ -349,7 +349,7 @@ def apply_chat_template_to_dataset(
if not isinstance(dataset, IterableDataset):
from multiprocessing import cpu_count
if num_proc is None or type(num_proc) is not int:
num_proc = min(cpu_count(), 8)
num_proc = max(1, cpu_count() // 3)
dataset_map_kwargs['num_proc'] = num_proc
dataset_map_kwargs['desc'] = f"Applying chat template to {final_format}"

6
studio/backend/utils/datasets/format_conversion.py
View file

@ -110,7 +110,7 @@ def standardize_chat_format(
from multiprocessing import cpu_count
if num_proc is None or type(num_proc) is not int:
num_proc = min(cpu_count(), 8)
num_proc = max(1, cpu_count() // 3)
dataset_map_kwargs['num_proc'] = num_proc
dataset_map_kwargs['desc'] = "Standardizing chat format"
@ -176,7 +176,7 @@ def convert_chatml_to_alpaca(dataset, batch_size=1000, num_proc=None):
from multiprocessing import cpu_count
if num_proc is None or type(num_proc) is not int:
num_proc = min(cpu_count(), 8)
num_proc = max(1, cpu_count() // 3)
dataset_map_kwargs['num_proc'] = num_proc
dataset_map_kwargs['desc'] = "Converting ChatML to Alpaca format"
@ -224,7 +224,7 @@ def convert_alpaca_to_chatml(dataset, batch_size=1000, num_proc=None):
from multiprocessing import cpu_count
if num_proc is None or type(num_proc) is not int:
num_proc = min(cpu_count(), 8)
num_proc = max(1, cpu_count() // 3)
dataset_map_kwargs['num_proc'] = num_proc
dataset_map_kwargs['desc'] = "Converting Alpaca to ChatML format"