ceos/unsloth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Synthetic data

Browse source
This commit is contained in:
Daniel Han 2025-04-30 07:31:27 -07:00
commit bf2e8a8724
3 changed files with 240 additions and 203 deletions
Download patch file Download diff file Expand all files Collapse all files

2
pyproject.toml
View file

@ -58,6 +58,7 @@ huggingface = [
"huggingface_hub",
"hf_transfer",
"unsloth[triton]",
"msgspec",
]
windows=[
"unsloth[huggingface]",
@ -370,6 +371,7 @@ colab-new = [
"hf_transfer",
"bitsandbytes>=0.43.3",
"unsloth[triton]",
"msgspec",
]
colab-no-deps = [
"accelerate>=0.34.1",

326
unsloth/dataprep/synthetic.py
View file

@ -28,218 +28,142 @@ import time
from unsloth_zoo.vllm_utils import load_vllm
from transformers import AutoConfig
def check_vllm_status():
try:
response = requests.get("http://localhost:8000/metrics")
if response.status_code == 200:
return True
except requests.exceptions.ConnectionError:
return False
pass
pass
from .sythetic_configs import (
synthetic_qa_config,
)
class SyntheticDataKit:
def async_load_vllm(
model_name = "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit",
max_seq_length = 2048,
gpu_memory_utilization = 0.9,
float8_kv_cache = False,
conservativeness = 1.0,
token = None,
):
config = AutoConfig.from_pretrained(
model_name,
token = token,
)
engine_args = load_vllm(
model_name = model_name,
config = config,
gpu_memory_utilization = gpu_memory_utilization,
max_seq_length = max_seq_length,
disable_log_stats = True,
float8_kv_cache = float8_kv_cache,
conservativeness = conservativeness,
return_args = True,
enable_lora = False,
)
if "device" in engine_args: del engine_args["device"]
if "model" in engine_args: del engine_args["model"]
if "compilation_config" in engine_args: del engine_args["compilation_config"]
def __init__()
def load_model(
model_name = "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit",
max_seq_length = 2048,
gpu_memory_utilization = 0.9,
float8_kv_cache = False,
conservativeness = 1.0,
token = None,
**kwargs,
):
assert(type(model_name) is str)
assert(type(max_seq_length) is int)
assert(type(gpu_memory_utilization) is float)
assert(type(float8_kv_cache) is bool)
assert(type(conservativeness) is float)
assert(token is None or type(token) is str)
subprocess_commands = [
"vllm", "serve", str(model_name),
]
for key, value in engine_args.items():
flag = key.replace("_", "-")
which = str(value).lower().replace("torch.", "")
if which == "true":
# Ignore --enforce-eager True
subprocess_commands += ["--" + flag,]
elif which == "false":
# Ignore flag
self.model_name = model_name
self.max_seq_length = max_seq_length
config = AutoConfig.from_pretrained(
model_name,
token = token,
)
engine_args = load_vllm(
model_name = model_name,
config = config,
gpu_memory_utilization = gpu_memory_utilization,
max_seq_length = max_seq_length,
disable_log_stats = True,
float8_kv_cache = float8_kv_cache,
conservativeness = conservativeness,
return_args = True,
enable_lora = False,
**kwargs,
)
if "device" in engine_args: del engine_args["device"]
if "model" in engine_args: del engine_args["model"]
if "compilation_config" in engine_args: del engine_args["compilation_config"]
subprocess_commands = [
"vllm", "serve", str(model_name),
]
for key, value in engine_args.items():
flag = key.replace("_", "-")
which = str(value).lower().replace("torch.", "")
if which == "true":
# Ignore --enforce-eager True
subprocess_commands += ["--" + flag,]
elif which == "false":
# Ignore flag
pass
else:
subprocess_commands += ["--" + flag, which,]
pass
print(subprocess_commands)
vllm_process = subprocess.Popen(
subprocess_commands,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
start_new_session = True,
)
ready_message_part = b"Starting vLLM API server on"
ready = False
while vllm_process.poll() is None:
output = vllm_process.stdout.readline()
if not output:
print("Stdout stream ended before readiness message detected.")
break
output_str = output.decode('utf-8', errors='ignore').strip()
print(f"vLLM STDOUT: {output_str}")
if ready_message_part in output:
print(f"\n--- vLLM Server Ready (Detected: '{ready_message_part.decode()}') ---")
ready = True
break
pass
else:
subprocess_commands += ["--" + flag, which,]
pass
if vllm_process is None:
raise RuntimeError("Unsloth: vllm_process failed to load!")
trial = 0
while not check_vllm_status():
if trial >= 100:
raise RuntimeError("Unsloth: vllm_process failed to load!")
trial += 1
time.sleep(1)
self.vllm_process = vllm_process
return
pass
print(subprocess_commands)
vllm_process = subprocess.Popen(
subprocess_commands,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
start_new_session = True,
)
ready_message_part = b"Starting vLLM API server on"
ready = False
while vllm_process.poll() is None:
output = vllm_process.stdout.readline()
if not output:
print("Stdout stream ended before readiness message detected.")
break
output_str = output.decode('utf-8', errors='ignore').strip()
print(f"vLLM STDOUT: {output_str}")
if ready_message_part in output:
print(f"\n--- vLLM Server Ready (Detected: '{ready_message_part.decode()}') ---")
ready = True
break
@staticmethod
def check_vllm_status():
try:
response = requests.get("http://localhost:8000/metrics")
if response.status_code == 200:
return True
except requests.exceptions.ConnectionError:
return False
pass
pass
if vllm_process is None:
raise RuntimeError("Unsloth: vllm_process failed to load!")
trial = 0
while not check_vllm_status():
if trial >= 100:
raise RuntimeError("Unsloth: vllm_process failed to load!")
trial += 1
time.sleep(1)
return vllm_process
pass
@staticmethod
def destroy_vllm(vllm_process):
print("Attempting to terminate the VLLM server gracefully...")
try:
vllm_process.terminate()
vllm_process.wait(timeout=10)
print("Server terminated gracefully.")
except subprocess.TimeoutExpired:
print("Server did not terminate gracefully after 10 seconds. Forcing kill...")
vllm_process.kill()
vllm_process.wait()
print("Server killed forcefully.")
except Exception as e:
print(f"An error occurred while trying to stop the process: {e}")
try:
if vllm_process.poll() is None:
print("Attempting forceful kill due to error...")
vllm_process.kill()
vllm_process.wait()
print("Server killed forcefully after error.")
except Exception as kill_e:
print(f"Error during forceful kill: {kill_e}")
for _ in range(10):
torch.cuda.empty_cache()
gc.collect()
pass
def destroy_vllm(vllm_process):
print("Attempting to terminate the VLLM server gracefully...")
try:
vllm_process.terminate()
vllm_process.wait(timeout=10)
print("Server terminated gracefully.")
except subprocess.TimeoutExpired:
print("Server did not terminate gracefully after 10 seconds. Forcing kill...")
vllm_process.kill()
vllm_process.wait()
print("Server killed forcefully.")
except Exception as e:
print(f"An error occurred while trying to stop the process: {e}")
try:
if vllm_process.poll() is None:
print("Attempting forceful kill due to error...")
vllm_process.kill()
vllm_process.wait()
print("Server killed forcefully after error.")
except Exception as kill_e:
print(f"Error during forceful kill: {kill_e}")
for _ in range(10):
torch.cuda.empty_cache()
gc.collect()
pass
synthetic_config_string = """\
# Master configuration file for Synthetic Data Kit
# Global paths configuration
paths:
# Input data locations
input:
pdf: "data/pdf"
html: "data/html"
youtube: "data/youtube"
docx: "data/docx"
ppt: "data/ppt"
txt: "data/txt"
# Output locations
output:
parsed: "data/output" # Where parsed text files are saved
generated: "data/generated" # Where generated content is saved
cleaned: "data/cleaned" # Where cleaned content is saved
final: "data/final" # Where final formatted content is saved
# VLLM server configuration
vllm:
api_base: "http://localhost:8000/v1" # Base URL for VLLM API
port: 8000 # Port for VLLM server
model: "{model_name}" # Default model to use
max_retries: 3 # Number of retries for API calls
retry_delay: 1.0 # Initial delay between retries (seconds)
# Ingest configuration
ingest:
default_format: "txt" # Default output format for parsed files
youtube_captions: "auto" # Options: "auto", "manual" - caption preference
# LLM generation parameters
generation:
temperature: {temperature} # Higher = more creative, lower = more deterministic
top_p: {top_p} # Nucleus sampling parameter
chunk_size: {chunk_size} # Size of text chunks for processing
overlap: {overlap} # Overlap between chunks to maintain context
max_tokens: {max_tokens} # Maximum tokens in LLM responses
num_pairs: {default_num_pairs} # Default number of QA pairs to generate
# Content cleanup parameters
cleanup:
threshold: {cleanup_threshold} # Default quality threshold (1-10)
batch_size: {cleanup_batch_size} # Number of items per batch for rating
temperature: {cleanup_temperature} # Temperature for rating (lower = more consistent)
# Format conversion parameters
format:
default: "jsonl" # Default output format
include_metadata: true # Include metadata in output files
pretty_json: true # Use indentation in JSON output
# Prompts for different tasks
prompts:
# Summary generation prompt
summary: |
Summarize this document in 3-5 sentences, focusing on the main topic and key concepts.
# QA pair generation prompt
qa_generation: |
Create {num_pairs} question-answer pairs from this text for LLM training.
Rules:
1. Questions must be about important facts in the text
2. Answers must be directly supported by the text
3. Return JSON format only:
[
{{
"question": "Question 1?",
"answer": "Answer 1."
}},
{{
"question": "Question 2?",
"answer": "Answer 2."
}}
]
Text:
{text}
# QA pair rating prompt
qa_rating: |
Rate each of these question-answer pairs for quality and return exactly this JSON format:
[
{{"question": "same question text", "answer": "same answer text", "rating": n}}
]
Where n is a number from 1-10.
DO NOT include any text outside of the JSON array, just return valid JSON:
{pairs}"""
def configure_synthetic_data_kit(
model_name = "unsloth/Llama-3.1-8B-Instruct-unsloth-bnb-4bit",

111
unsloth/dataprep/synthetic_configs.py Normal file
View file

@ -0,0 +1,111 @@
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
synthetic_qa_config = """\
# Master configuration file for Synthetic Data Kit
# Global paths configuration
paths:
# Input data locations
input:
pdf: "data/pdf"
html: "data/html"
youtube: "data/youtube"
docx: "data/docx"
ppt: "data/ppt"
txt: "data/txt"
# Output locations
output:
parsed: "data/output" # Where parsed text files are saved
generated: "data/generated" # Where generated content is saved
cleaned: "data/cleaned" # Where cleaned content is saved
final: "data/final" # Where final formatted content is saved
# VLLM server configuration
vllm:
api_base: "http://localhost:8000/v1" # Base URL for VLLM API
port: 8000 # Port for VLLM server
model: "{model_name}" # Default model to use
max_retries: 3 # Number of retries for API calls
retry_delay: 1.0 # Initial delay between retries (seconds)
# Ingest configuration
ingest:
default_format: "txt" # Default output format for parsed files
youtube_captions: "auto" # Options: "auto", "manual" - caption preference
# LLM generation parameters
generation:
temperature: {temperature} # Higher = more creative, lower = more deterministic
top_p: {top_p} # Nucleus sampling parameter
chunk_size: {chunk_size} # Size of text chunks for processing
overlap: {overlap} # Overlap between chunks to maintain context
max_tokens: {max_tokens} # Maximum tokens in LLM responses
num_pairs: {default_num_pairs} # Default number of QA pairs to generate
# Content cleanup parameters
cleanup:
threshold: {cleanup_threshold} # Default quality threshold (1-10)
batch_size: {cleanup_batch_size} # Number of items per batch for rating
temperature: {cleanup_temperature} # Temperature for rating (lower = more consistent)
# Format conversion parameters
format:
default: "jsonl" # Default output format
include_metadata: true # Include metadata in output files
pretty_json: true # Use indentation in JSON output
# Prompts for different tasks
prompts:
# Summary generation prompt
summary: |
Summarize this document in 3-5 sentences, focusing on the main topic and key concepts.
# QA pair generation prompt
qa_generation: |
Create {num_pairs} question-answer pairs from this text for LLM training.
Rules:
1. Questions must be about important facts in the text
2. Answers must be directly supported by the text
3. Return JSON format only:
[
{{
"question": "Question 1?",
"answer": "Answer 1."
}},
{{
"question": "Question 2?",
"answer": "Answer 2."
}}
]
Text:
{text}
# QA pair rating prompt
qa_rating: |
Rate each of these question-answer pairs for quality and return exactly this JSON format:
[
{{"question": "same question text", "answer": "same answer text", "rating": n}}
]
Where n is a number from 1-10.
DO NOT include any text outside of the JSON array, just return valid JSON:
{pairs}"""