diff --git a/pyproject.toml b/pyproject.toml index 0a3cfa1f79..510e186cde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,18 +37,18 @@ triton = [ ] huggingface = [ - "unsloth_zoo>=2025.9.11", + "unsloth_zoo>=2025.9.13", "packaging", "tyro", "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,<=4.56.2", - "datasets>=3.4.1,<4.0.0", + "datasets>=3.4.1,!=4.0.*,!=4.1.0", "sentencepiece>=0.2.0", "tqdm", "psutil", "wheel>=0.42.0", "numpy", "accelerate>=0.34.1", - "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0,<=0.23.0", "peft>=0.7.1,!=0.11.0", "protobuf", "huggingface_hub>=0.34.0", @@ -453,11 +453,11 @@ colab-ampere-torch220 = [ "flash-attn>=2.6.3", ] colab-new = [ - "unsloth_zoo>=2025.9.11", + "unsloth_zoo>=2025.9.13", "packaging", "tyro", - "transformers>=4.51.3,!=4.47.0,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,<=4.56.2", - "datasets>=3.4.1,<4.0.0", + "transformers>=4.51.3,!=4.52.0,!=4.52.1,!=4.52.2,!=4.52.3,!=4.53.0,!=4.54.0,!=4.55.0,!=4.55.1,<=4.56.2", + "datasets>=3.4.1,!=4.0.*,!=4.1.0", "sentencepiece>=0.2.0", "tqdm", "psutil", @@ -471,7 +471,7 @@ colab-new = [ ] colab-no-deps = [ "accelerate>=0.34.1", - "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0", + "trl>=0.7.9,!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,!=0.15.0,!=0.19.0,<=0.23.0", "peft>=0.7.1", "xformers", "bitsandbytes>=0.45.5", diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py index 5af8c756e3..2df9b878d0 100644 --- a/unsloth/models/_utils.py +++ b/unsloth/models/_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2025.9.9" +__version__ = "2025.9.10" __all__ = [ "SUPPORTS_BFLOAT16", diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py index 98396bb754..3bcab1ce87 100644 --- a/unsloth/models/loader.py +++ b/unsloth/models/loader.py @@ -662,10 +662,15 @@ class FastModel(FastBaseModel): ) model_types_all = ",".join(model_types) + "," - # Check versions + # Save model types and loading method lowered_model_name = model_name.lower() - if os.environ.get("UNSLOTH_MODEL_NAME", "") == "": - os.environ["UNSLOTH_MODEL_NAME"] = lowered_model_name + string = os.environ.get("UNSLOTH_MODEL_NAME", "") + model_types_all + if load_in_4bit: string += "_load_in_4bit_" + if load_in_8bit: string += "_load_in_8bit_" + if load_in_16bit: string += "_load_in_16bit_" + os.environ["UNSLOTH_MODEL_NAME"] = string + + # Check versions LATEST = '\nPlease use transformers via `pip install --no-deps git+https://github.com/huggingface/transformers.git`' NIGHTLY = '\nPlease use nightly transformers via pip install --upgrade "transformers>=4.49.0"`' # Pixtral diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py index ec81106890..a207514e72 100644 --- a/unsloth/models/rl_replacements.py +++ b/unsloth/models/rl_replacements.py @@ -27,6 +27,7 @@ import inspect from collections import defaultdict from unsloth_zoo.rl_replacements import RL_REPLACEMENTS, left_pack_padding from unsloth import DEVICE_TYPE +import textwrap RL_EXTRA_ARGS = defaultdict(list) RL_FUNCTIONS = defaultdict(list) @@ -295,12 +296,59 @@ def grpo_trainer__generate_and_score_completions(function_name, function): if self.use_vllm:""" function = function.replace(replace_part, new_replacement) - return function pass RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer__generate_and_score_completions) +# Fix {"reasoning_effort" : "high"} not applied +def grpo_trainer_fix_maybe_apply_chat_template(function_name, function): + spaces = function.find("def ") + if spaces % 4 != 0: return function + spaces += 4 + replacement = """ + _chat_template_ = getattr(self.processing_class, "chat_template", None) + if _chat_template_ is None: _chat_template_ = "" + _supported_keys_ = set(("prompt", "chosen", "rejected", "completion", "messages", "label")) + + prompts_text = [] + for _example_ in __INPUTS__REPLACEMENT__: + _tokenizer_kwargs_ = {} + if type(_example_) is not dict: + _example_ = {"prompt": _example_} + _left_keys_ = _example_.keys() - _supported_keys_ + for k in _left_keys_: + if k in _chat_template_: + v = _example_[k] + if type(v) is str: + _tokenizer_kwargs_[k] = v + _x_ = maybe_apply_chat_template(_example_, self.processing_class, **_tokenizer_kwargs_)["prompt"] + prompts_text.append(_x_) + """ + replacement = textwrap.dedent(replacement).strip() + replacement = textwrap.indent(replacement, spaces*" ") + replacement = f"\n{replacement}\n" + what = 'prompts_text = [maybe_apply_chat_template(example, self.processing_class)["prompt"] for example in inputs]' + function = function.replace(what, replacement.replace("__INPUTS__REPLACEMENT__", "inputs")) + + """prompts_text = [ + maybe_apply_chat_template({"prompt": prompt}, self.processing_class)["prompt"] for prompt in prompts + ]""" + function = re.sub( + r"prompts_text = \["\ + r"[\s]{0,}"\ + r"maybe_apply_chat_template\(\{[\"\']prompt[\"\'][\s]{0,}\:[\s]{0,}prompt[\s]{0,}\}[\s]{0,}\,[\s]{0,}self\.processing_class\)"\ + r"\[[\"\']prompt[\"\']\] for prompt in prompts"\ + r"[\s]{0,}"\ + r"\]", + replacement.replace("__INPUTS__REPLACEMENT__", "prompts"), + function, + ) + return function +pass +RL_FUNCTIONS["grpo_trainer"].append(grpo_trainer_fix_maybe_apply_chat_template) + + # Remove _move_model_to_vllm def grpo_trainer__move_model_to_vllm(function_name, function): if function_name != "_move_model_to_vllm": return function diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py index 8a7b14478f..ae76c573d0 100644 --- a/unsloth/models/vision.py +++ b/unsloth/models/vision.py @@ -524,8 +524,8 @@ class FastBaseModel: quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING[quantization_config["quant_method"]] quantizer_kwargs = {} # We cannot dequantize since gpt-oss-20b MXFP4 will now be gpt-oss-20b-BF16 - # if "dequantize" in inspect.signature(quantizer).parameters: - # quantizer_kwargs["dequantize"] = True + if load_in_16bit and "dequantize" in inspect.signature(quantizer).parameters: + quantizer_kwargs["dequantize"] = True quantization_config = quantizer.from_dict(quantization_config, **quantizer_kwargs) kwargs["quantization_config"] = quantization_config pass @@ -549,7 +549,7 @@ class FastBaseModel: # attn_implementation = attn_implementation, **kwargs, ) - if hasattr(model, 'generate'): + if hasattr(model, "generate"): model.fast_generate = model.generate model.fast_generate_batches = error_out_no_vllm if offload_embedding: @@ -612,8 +612,17 @@ class FastBaseModel: llm = load_vllm(**load_vllm_kwargs) # Convert to HF format - _, quant_state_dict = get_vllm_state_dict(llm, config = model_config, is_vision_model = True) - model = convert_vllm_to_huggingface(quant_state_dict, model_config, dtype, bnb_config, is_vision_model = True) + _, quant_state_dict = get_vllm_state_dict( + llm, + config = model_config, + is_vision_model = True, + ) + model = convert_vllm_to_huggingface( + quant_state_dict, + model_config, + dtype, bnb_config, + is_vision_model = True, + ) model.vllm_engine = llm model.fast_generate = model.vllm_engine.generate model.fast_generate_batches = functools.partial(generate_batches, model.vllm_engine) @@ -753,52 +762,6 @@ class FastBaseModel: return model, tokenizer pass - @staticmethod - def pre_compile_for_inference(model_type, model, tokenizer): - """ - We need to invoke torch.compile to save VRAM usage and make it faster downstream. - Sometimes torch.compile can use 3GB weirdly on large batches, then it goes down to <1GB. - So we invoke torch.compile on short batches to reduce VRAM usage. - """ - if model_type is None or model is None or tokenizer is None: return - if str(model_type).lower() not in PRE_COMPILE_INFERENCE: return - if getattr(tokenizer, "chat_template", None) is None: return - # Check if already compiled and exit - for module in model.modules(): - if hasattr(module, "_pre_compiled_for_inference"): return - pass - print(f"🦥 Unsloth: Pre compiling {model_type.title()} model for faster inference - this might take 3 minutes or so!") - print("========= Pre compiling model for faster inference. Please be patient thank you! =========") - # Do single inference - messages = [ - [ - {"role": "user", "content": f"What is 1+1 equal to?"}, - ], - ]*1 - inputs = tokenizer.apply_chat_template( - messages, - add_generation_prompt = True, - return_tensors = "pt", - return_dict = True, - ).to(model.device) - _ = model.generate(**inputs, max_new_tokens = 1) - # Do batched inference - messages = [ - [ - {"role": "user", "content": f"1+1"}, - ], - ]*4 - inputs = tokenizer.apply_chat_template( - messages, - add_generation_prompt = True, - return_tensors = "pt", - return_dict = True, - ).to(model.device) - _ = model.generate(**inputs, max_new_tokens = 2) - # Set we already pre compiled - model._pre_compiled_for_inference = True - pass - @staticmethod def get_peft_model( model, @@ -902,7 +865,11 @@ class FastBaseModel: # Enable gradients on modules which are trainable requires_grad_for_gradient_checkpointing(model) trust_remote_code = getattr(model, "_unsloth_trust_remote_code", False) - model = FastBaseModel.post_patch_model(model, use_gradient_checkpointing, trust_remote_code = trust_remote_code) + model = FastBaseModel.post_patch_model( + model, + use_gradient_checkpointing = use_gradient_checkpointing, + trust_remote_code = trust_remote_code, + ) model.max_seq_length = max_seq_length # Save to modules as well for module in model.modules(): @@ -998,14 +965,15 @@ class FastBaseModel: m.for_inference = functools.partial(FastBaseModel.for_inference, m) m = m.model # Set weight[padding_idx] = 0 - with torch.no_grad(): - for name, module in model.named_modules(): - if type(module) is torch.nn.Embedding: - if getattr(module, "weight", None) is not None and getattr(module, "padding_idx", None) is not None: - if module.padding_idx < module.weight.shape[0]: - module.weight[module.padding_idx] = 0 - # Patch for torch.compiled inference - # FastBaseModel.pre_compile_for_inference(model_type, model, tokenizer) + # Only do this if tokenizer is defined since eos_token == pad_token sometimes! + pad_token_id = getattr(tokenizer, "pad_token_id", None) + if tokenizer is not None and getattr(tokenizer, "eos_token_id", None) != pad_token_id: + with torch.no_grad(): + for name, module in model.named_modules(): + if type(module) is torch.nn.Embedding: + if getattr(module, "weight", None) is not None and getattr(module, "padding_idx", None) is not None: + if module.padding_idx == pad_token_id and module.padding_idx < module.weight.shape[0]: + module.weight[module.padding_idx] = 0 return model pass