From 2e07ee2346fe29da0a94a2e047d79ecf0c331de3 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Thu, 1 May 2025 07:42:08 -0700 Subject: [PATCH] Nightly (#2448) * move float32 * Ensure trust_remote_code propegates down to unsloth_compile_transformers (#2075) * Update _utils.py * Show both `peft_error` and `autoconfig_error`, not just `autoconfig_error` (#2080) When loading a PEFT model fails, only the `autoconfig_error` is shown. Instead of the `peft_error`, which is what really matters when we're trying to load a PEFT adapter, the user will see something like this: ``` RuntimeError: Unrecognized model in my_model. Should have a `model_type` key in its config.json, or contain one of the following strings in its name: albert, align, altclip, ... ``` This PR just changes it so `autoconfig_error` and `peft_error` are both displayed. * fix error message (#2046) * Update vision.py * Update _utils.py * Update pyproject.toml * Update __init__.py * Update __init__.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update rl_replacements.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update rl_replacements.py * Update vision.py * Update rl_replacements.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Remove double generate patch * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update mapper.py * Update vision.py * fix: config.torch_dtype in LlamaModel_fast_forward_inference (#2091) * fix: config.torch_dtype in LlamaModel_fast_forward_inference * Update llama.py * update for consistency --------- Co-authored-by: Daniel Han * versioning * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * model_type_arch * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * check * Update _utils.py * Update loader.py * Update loader.py * Remove prints * Update README.md typo * Update _utils.py * Update _utils.py * versioning * Update _utils.py * Update _utils.py * Update _utils.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update llama.py * Update vision.py * HF Transfer * fix(utils): add missing importlib import to fix NameError (#2134) This commit fixes a NameError that occurs when `importlib` is referenced in _utils.py without being imported, especially when UNSLOTH_USE_MODELSCOPE=1 is enabled. By adding the missing import statement, the code will no longer throw a NameError. * Add QLoRA Train and Merge16bit Test (#2130) * add reference and unsloth lora merging tests * add test / dataset printing to test scripts * allow running tests from repo root * add qlora test readme * more readme edits * ruff formatting * additional readme comments * forgot to add actual tests * add apache license * Update pyproject.toml * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update loader.py * Update loader.py * Revert * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Update vision.py * Bug fix * Update mapper.py * check SDPA for Mistral 3, Pixtral * Update vision.py * Versioning * Update rl_replacements.py * Update README.md * add model registry * move hf hub utils to unsloth/utils * refactor global model info dicts to dataclasses * fix dataclass init * fix llama registration * remove deprecated key function * start registry reog * add llama vision * quant types -> Enum * remap literal quant types to QuantType Enum * add llama model registration * fix quant tag mapping * add qwen2.5 models to registry * add option to include original model in registry * handle quant types per model size * separate registration of base and instruct llama3.2 * add QwenQVQ to registry * add gemma3 to registry * add phi * add deepseek v3 * add deepseek r1 base * add deepseek r1 zero * add deepseek distill llama * add deepseek distill models * remove redundant code when constructing model names * add mistral small to registry * rename model registration methods * rename deepseek registration methods * refactor naming for mistral and phi * add global register models * refactor model registration tests for new registry apis * add model search method * remove deprecated registration api * add quant type test * add registry readme * make llama registration more specific * clear registry when executing individual model registration file * more registry readme updates * Update _auto_install.py * Llama4 * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Synthetic data * Update mapper.py * Xet and Synthetic * Update synthetic.py * Update loader.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update pyproject.toml * Delete .gitignore * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update synthetic.py * Update _utils.py * Update pyproject.toml * Update synthetic.py * Update synthetic.py --------- Co-authored-by: Xander Hawthorne <167850078+CuppaXanax@users.noreply.github.com> Co-authored-by: Isaac Breen Co-authored-by: Kareem <81531392+KareemMusleh@users.noreply.github.com> Co-authored-by: lurf21 <93976703+lurf21@users.noreply.github.com> Co-authored-by: Jack Shi Wei Lun <87535974+jackswl@users.noreply.github.com> Co-authored-by: naliazheli Co-authored-by: jeromeku Co-authored-by: Michael Han <107991372+shimmyshimmer@users.noreply.github.com> --- pyproject.toml | 4 +-- unsloth/dataprep/synthetic.py | 67 +++++++++++++++++++++-------------- unsloth/models/_utils.py | 2 +- 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2b258ba4cf..6866317fb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.4.3", + "unsloth_zoo>=2025.4.4", "packaging", "tyro", "transformers>=4.46.1,!=4.47.0", @@ -351,7 +351,7 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.4.3", + "unsloth_zoo>=2025.4.4", "packaging", "tyro", "transformers>=4.46.1,!=4.47.0", diff --git a/unsloth/dataprep/synthetic.py b/unsloth/dataprep/synthetic.py index 2f4a85f143..100044b54b 100644 --- a/unsloth/dataprep/synthetic.py +++ b/unsloth/dataprep/synthetic.py @@ -18,13 +18,16 @@ __all__ = [ import subprocess import time import os +os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" import requests import torch import gc import time -from unsloth_zoo.vllm_utils import load_vllm -from transformers import AutoConfig, AutoTokenizer -import signal +from unsloth_zoo.vllm_utils import ( + load_vllm, + patch_vllm, +) +import numpy as np from .synthetic_configs import ( synthetic_qa_config, @@ -51,6 +54,7 @@ class SyntheticDataKit: self.model_name = model_name self.max_seq_length = max_seq_length + from transformers import AutoConfig, AutoTokenizer self.config = AutoConfig.from_pretrained( model_name, token = token, @@ -59,6 +63,7 @@ class SyntheticDataKit: model_name, token = token, ) + patch_vllm() engine_args = load_vllm( model_name = model_name, config = self.config, @@ -69,23 +74,23 @@ class SyntheticDataKit: conservativeness = conservativeness, return_args = True, enable_lora = False, + use_bitsandbytes = False, **kwargs, ) if "device" in engine_args: del engine_args["device"] if "model" in engine_args: del engine_args["model"] - if "compilation_config" in engine_args: del engine_args["compilation_config"] subprocess_commands = [ "vllm", "serve", str(model_name), ] for key, value in engine_args.items(): flag = key.replace("_", "-") - which = str(value).lower().replace("torch.", "") - if which == "true": + which = str(value).replace("torch.", "") + if which == "True": # Ignore --enforce-eager True subprocess_commands += ["--" + flag,] - elif which == "false": + elif which == "False": # Ignore flag pass else: @@ -190,34 +195,42 @@ class SyntheticDataKit: def __exit__(self, *exc): self.cleanup() def __del__(self): self.cleanup() - def truncate(self, filename = None): - # Truncates by summary and max generation + def chunk_data(self, filename = None): + # Chunks data by max tokens and generation length assert(filename is not None) assert(os.path.exists(filename)) assert(hasattr(self, "tokenizer")) + if not hasattr(self, "max_seq_length"): + raise RuntimeError("Please use SynthetidDataKit.from_pretrained(...) first!") + if not hasattr(self, "overlap") or not hasattr(self, "max_generation_tokens"): + raise RuntimeError("Please use prepare_qa_generation first!") with open(filename, "r") as f: text = f.read() - max_tokens = self.max_seq_length - self.max_generation_tokens*2 - 2 - input_ids = self.tokenizer(text).input_ids - length = len(text) - original_length = len(text) - original_n_tokens = len(input_ids) + max_tokens = self.max_seq_length - self.max_generation_tokens*2 - 128 # -128 to reduce errors + if max_tokens <= 5: + raise RuntimeError("Generation length is way too long!") + input_ids = self.tokenizer(text, add_special_tokens = False).input_ids - if len(input_ids) > max_tokens: - # Will fix later, but for now we simply naively truncate by ratios - length = original_length - while True: - input_ids = self.tokenizer(text[:length]).input_ids - if len(input_ids) < max_tokens or length == 0: break - length = length * (max_tokens/len(input_ids)) - length = max(int(length), 0) - pass - print(f"Unsloth: Will truncate your data which has {original_n_tokens} tokens to {len(input_ids)} tokens.") + # Get left and right boundaries + length = len(input_ids) + n_chunks = int(np.ceil(length / (max_tokens - self.overlap))) + boundaries = np.ceil(np.linspace(0, length - self.overlap, n_chunks)).astype(int) + boundaries = np.stack((boundaries[:-1], (boundaries + self.overlap)[1:])).T + boundaries = np.minimum(boundaries, length).tolist() - with open(filename, "w") as f: f.write(text[:length]) + # Get extension of filename like .txt + filename, extension = os.path.splitext(filename) + if filename.endswith("/"): filename = filename[:-1] + + all_filenames = [] + for i, (left, right) in enumerate(boundaries): + chunked_text = self.tokenizer.decode(input_ids[left : right]) + new_filename = f"{filename}_{i}{extension}" + all_filenames.append(new_filename) + with open(new_filename, "w") as f: f.write(chunked_text) pass - return filename, length + return all_filenames pass def prepare_qa_generation( @@ -258,5 +271,7 @@ class SyntheticDataKit: .replace("{cleanup_temperature}", str(cleanup_temperature)) with open("synthetic_data_kit_config.yaml", "w") as f: f.write(config) + + self.overlap = overlap pass pass diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index ed8a2ade64..d3b8969ba6 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.4.3" +__version__ = "2025.4.4" __all__ = [ "SUPPORTS_BFLOAT16",