From ed9709bdcf1fce3881a34a679f96c5d4cfcf86e8 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 30 Apr 2025 08:37:56 -0700 Subject: [PATCH] Xet and Synthetic --- pyproject.toml | 4 +- unsloth/dataprep/synthetic.py | 132 ++++++++++++++++++-------- unsloth/dataprep/synthetic_configs.py | 20 ++-- 3 files changed, 102 insertions(+), 54 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e25af70f87..dbd4ff96b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ huggingface = [ "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,<=0.15.2", "peft>=0.7.1,!=0.11.0", "protobuf<4.0.0", - "huggingface_hub", + "huggingface_hub[hf_xet] >= 0.30.0", "hf_transfer", "unsloth[triton]", "msgspec", @@ -367,7 +367,7 @@ colab-new = [ "wheel>=0.42.0", "numpy", "protobuf<4.0.0", - "huggingface_hub", + "huggingface_hub[hf_xet] >= 0.30.0", "hf_transfer", "bitsandbytes>=0.43.3", "unsloth[triton]", diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 827187b7d5..5dc37bac70 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -26,16 +26,17 @@ import torch import gc import time from unsloth_zoo.vllm_utils import load_vllm -from transformers import AutoConfig +from transformers import AutoConfig, AutoTokenizer +import signal +import atexit +import weakref from .sythetic_configs import ( synthetic_qa_config, ) class SyntheticDataKit: - - def __init__() - def load_model( + def __init__( model_name = "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit", max_seq_length = 2048, gpu_memory_utilization = 0.9, @@ -54,13 +55,17 @@ class SyntheticDataKit: self.model_name = model_name self.max_seq_length = max_seq_length - config = AutoConfig.from_pretrained( + self.config = AutoConfig.from_pretrained( + model_name, + token = token, + ) + self.tokenizer = AutoTokenizer.from_pretrained( model_name, token = token, ) engine_args = load_vllm( model_name = model_name, - config = config, + config = self.config, gpu_memory_utilization = gpu_memory_utilization, max_seq_length = max_seq_length, disable_log_stats = True, @@ -70,6 +75,7 @@ class SyntheticDataKit: enable_lora = False, **kwargs, ) + if "device" in engine_args: del engine_args["device"] if "model" in engine_args: del engine_args["model"] if "compilation_config" in engine_args: del engine_args["compilation_config"] @@ -89,13 +95,15 @@ class SyntheticDataKit: else: subprocess_commands += ["--" + flag, which,] pass - print(subprocess_commands) vllm_process = subprocess.Popen( subprocess_commands, stdout = subprocess.PIPE, stderr = subprocess.PIPE, start_new_session = True, ) + atexit.register(self.destroy_vllm) + self._finalizer = weakref.finalize(self, self.destroy_vllm) + ready_message_part = b"Starting vLLM API server on" ready = False while vllm_process.poll() is None: @@ -134,8 +142,10 @@ class SyntheticDataKit: pass pass - @staticmethod - def destroy_vllm(vllm_process): + def destroy_vllm(self): + if not hasattr(self, vllm_process): return + + vllm_process = self.vllm_process print("Attempting to terminate the VLLM server gracefully...") try: vllm_process.terminate() @@ -161,40 +171,78 @@ class SyntheticDataKit: gc.collect() pass - - - - -def configure_synthetic_data_kit( - model_name = "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit", - output_folder = "synthetic_data_output", - temperature = 0.7, - top_p = 0.95, - chunk_size = 4000, - overlap = 200, - max_tokens = 512, - default_num_pairs = 25, - cleanup_threshold = 1.0, - cleanup_batch_size = 4, - cleanup_temperature = 0.3, -): - locations = "pdf,html,youtube,docx,ppt,txt,output,generated,cleaned,final" - locations = locations.split(",") - for path in locations: - os.makedirs(os.path.join(output_folder, path), exist_ok = True) + def __enter__(self): return self + def __exit__(self, *exc): self.destroy_vllm() + def __del__(self): + try: + self.destroy_vllm() + except Exception: + pass pass - config = synthetic_config_string\ - .replace("{model_name}", str(model_name))\ - .replace("{temperature}", str(temperature))\ - .replace("{top_p}", str(top_p))\ - .replace("{chunk_size}", str(chunk_size))\ - .replace("{overlap}", str(overlap))\ - .replace("{max_tokens}", str(max_tokens))\ - .replace("{default_num_pairs}", str(default_num_pairs))\ - .replace("{cleanup_threshold}", str(cleanup_threshold))\ - .replace("{cleanup_batch_size}", str(cleanup_batch_size))\ - .replace("{cleanup_temperature}", str(cleanup_temperature)) + def truncate(self, filename = None): + # Truncates by summary and max generation + assert(filename is not None) + assert(os.path.exists(filename)) + assert(hasattr(self, "tokenizer")) - return config + with open(filename, "r") as f: text = f.read() + + max_tokens = self.max_seq_length - self.max_generation_tokens + self.max_generation_tokens + 2 + input_ids = self.tokenizer(text).input_ids + length = len(text) + original_length = len(text) + original_n_tokens = len(input_ids) + + if len(input_ids) > max_tokens: + # Will fix later, but for now we simply naively truncate by 10% increments + ratio = 0.9 + length = original_length + while True: + input_ids = self.tokenizer(text[:length]).input_ids + if len(input_ids) < max_tokens or length == 0: break + length = int(original_length * ratio) + length = max(length, 0) + ratio -= 0.1 + pass + print(f"Unsloth: Will truncate your data which has {original_n_tokens} tokens to {len(input_ids)} tokens.") + + with open(filename, "w") as f: text = f.read() + pass + return filename + pass + + def configure_synthetic_data_kit( + output_folder = "synthetic_data_output", + max_generation_tokens = 512, + temperature = 0.7, + top_p = 0.95, + chunk_size = 4000, + overlap = 200, + default_num_pairs = 25, + cleanup_threshold = 1.0, + cleanup_batch_size = 4, + cleanup_temperature = 0.3, + ): + locations = "pdf,html,youtube,docx,ppt,txt,output,generated,cleaned,final" + locations = locations.split(",") + for path in locations: + os.makedirs(os.path.join(output_folder, path), exist_ok = True) + pass + + config = synthetic_config_string\ + .replace("{data_output_location}", str(output_folder))\ + .replace("{model_name}", str(model_name))\ + .replace("{temperature}", str(temperature))\ + .replace("{top_p}", str(top_p))\ + .replace("{chunk_size}", str(chunk_size))\ + .replace("{overlap}", str(overlap))\ + .replace("{max_tokens}", str(max_generation_tokens))\ + .replace("{default_num_pairs}", str(default_num_pairs))\ + .replace("{cleanup_threshold}", str(cleanup_threshold))\ + .replace("{cleanup_batch_size}", str(cleanup_batch_size))\ + .replace("{cleanup_temperature}", str(cleanup_temperature)) + + with open("synthetic_data_kit_config.yaml", "w") as f: f.write(config) + pass pass diff --git a/unsloth/dataprep/synthetic_configs.py b/unsloth/dataprep/synthetic_configs.py index 614cf4cfe7..f428177528 100644 --- a/unsloth/dataprep/synthetic_configs.py +++ b/unsloth/dataprep/synthetic_configs.py @@ -19,19 +19,19 @@ synthetic_qa_config = """\ paths: # Input data locations input: - pdf: "data/pdf" - html: "data/html" - youtube: "data/youtube" - docx: "data/docx" - ppt: "data/ppt" - txt: "data/txt" + pdf: "{data_output_location}/pdf" + html: "{data_output_location}/html" + youtube: "{data_output_location}/youtube" + docx: "{data_output_location}/docx" + ppt: "{data_output_location}/ppt" + txt: "{data_output_location}/txt" # Output locations output: - parsed: "data/output" # Where parsed text files are saved - generated: "data/generated" # Where generated content is saved - cleaned: "data/cleaned" # Where cleaned content is saved - final: "data/final" # Where final formatted content is saved + parsed: "{data_output_location}/output" # Where parsed text files are saved + generated: "{data_output_location}/generated" # Where generated content is saved + cleaned: "{data_output_location}/cleaned" # Where cleaned content is saved + final: "{data_output_location}/final" # Where final formatted content is saved # VLLM server configuration vllm: